diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index da22bc7edf..55df548cf4 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -21,6 +21,7 @@ "workflow" ], "skills": "./skills/", + "hooks": "./hooks/hooks-codex.json", "interface": { "displayName": "Superpowers", "shortDescription": "Planning, TDD, debugging, and delivery workflows for coding agents", diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index b007cf60de..85c229c095 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -19,7 +19,5 @@ "workflows" ], "skills": "./skills/", - "agents": "./agents/", - "commands": "./commands/", "hooks": "./hooks/hooks-cursor.json" } diff --git a/.gitignore b/.gitignore index 1c25a50ba5..981f4bc382 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,9 @@ node_modules/ inspo triage/ + +# Eval harness — drill ships its own gitignore at evals/.gitignore; +# these are belt-and-suspenders entries for tools that don't recurse. +evals/results/ +evals/.venv/ +evals/.env diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..6213ceead5 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "evals"] + path = evals + url = git@github.com:prime-radiant-inc/superpowers-evals.git diff --git a/.opencode/INSTALL.md b/.opencode/INSTALL.md index 94c34151bb..080f043f8f 100644 --- a/.opencode/INSTALL.md +++ b/.opencode/INSTALL.md @@ -45,7 +45,7 @@ Use OpenCode's native `skill` tool: ``` use skill tool to list skills -use skill tool to load superpowers/brainstorming +use skill tool to load brainstorming ``` ## Updating @@ -98,11 +98,16 @@ Then use the installed package path in `opencode.json`: ### Tool mapping -When skills reference Claude Code tools: -- `TodoWrite` → `todowrite` -- `Task` with subagents → `@mention` syntax -- `Skill` tool → OpenCode's native `skill` tool -- File operations → your native tools +Skills speak in actions ("create a todo", "dispatch a subagent", "read a file"). On OpenCode these resolve to: + +- "Create a todo" / "mark complete in todo list" → `todowrite` +- `Subagent (general-purpose):` template → `task` tool with `subagent_type: "general"` (or `"explore"` for codebase exploration) +- "Invoke a skill" → OpenCode's native `skill` tool +- "Read a file" → `read` +- "Create a file" / "edit a file" / "delete a file" → `apply_patch` +- "Run a shell command" → `bash` +- "Search file contents" / "find files by name" → `grep`, `glob` +- "Fetch a URL" → `webfetch` ## Getting Help diff --git a/.opencode/plugins/superpowers.js b/.opencode/plugins/superpowers.js index f2b95f2353..423e5ed513 100644 --- a/.opencode/plugins/superpowers.js +++ b/.opencode/plugins/superpowers.js @@ -1,7 +1,7 @@ /** * Superpowers plugin for OpenCode.ai * - * Injects superpowers bootstrap context via system prompt transform. + * Injects superpowers bootstrap context via message transform. * Auto-registers skills directory via config hook (no symlinks needed). */ @@ -74,11 +74,15 @@ export const SuperpowersPlugin = async ({ client, directory }) => { const { content } = extractAndStripFrontmatter(fullContent); const toolMapping = `**Tool Mapping for OpenCode:** -When skills reference tools you don't have, substitute OpenCode equivalents: -- \`TodoWrite\` → \`todowrite\` -- \`Task\` tool with subagents → Use OpenCode's subagent system (@mention) -- \`Skill\` tool → OpenCode's native \`skill\` tool -- \`Read\`, \`Write\`, \`Edit\`, \`Bash\` → Your native tools +When skills request actions, substitute OpenCode equivalents: +- Create or update todos → \`todowrite\` +- \`Subagent (general-purpose):\` → \`task\` with \`subagent_type: "general"\` +- Invoke a skill → OpenCode's native \`skill\` tool +- Read files → \`read\` +- Create, edit, or delete files → \`apply_patch\` +- Run shell commands → \`bash\` +- Search files → \`grep\`, \`glob\` +- Fetch a URL → \`webfetch\` Use OpenCode's native \`skill\` tool to list and load skills.`; diff --git a/.pi/extensions/superpowers.ts b/.pi/extensions/superpowers.ts new file mode 100644 index 0000000000..a978e80ee0 --- /dev/null +++ b/.pi/extensions/superpowers.ts @@ -0,0 +1,121 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; + +const EXTREMELY_IMPORTANT_MARKER = ""; +const BOOTSTRAP_MARKER = "superpowers:using-superpowers bootstrap for pi"; + +const extensionDir = dirname(fileURLToPath(import.meta.url)); +const packageRoot = resolve(extensionDir, "../.."); +const skillsDir = resolve(packageRoot, "skills"); +const bootstrapSkillPath = resolve(skillsDir, "using-superpowers", "SKILL.md"); + +let cachedBootstrap: string | null | undefined; + +export default function superpowersPiExtension(pi: ExtensionAPI) { + let injectBootstrap = true; + + pi.on("resources_discover", async () => ({ + skillPaths: [skillsDir], + })); + + pi.on("session_start", async () => { + injectBootstrap = true; + }); + + pi.on("session_compact", async () => { + injectBootstrap = true; + }); + + pi.on("agent_end", async () => { + injectBootstrap = false; + }); + + pi.on("context", async (event) => { + if (!injectBootstrap) return; + if (event.messages.some(messageContainsBootstrap)) return; + + const bootstrap = getBootstrapContent(); + if (!bootstrap) return; + + const bootstrapMessage = { + role: "user" as const, + content: [{ type: "text" as const, text: bootstrap }], + timestamp: Date.now(), + }; + + const insertAt = firstNonCompactionSummaryIndex(event.messages); + return { + messages: [ + ...event.messages.slice(0, insertAt), + bootstrapMessage, + ...event.messages.slice(insertAt), + ], + }; + }); +} + +function getBootstrapContent(): string | null { + if (cachedBootstrap !== undefined) return cachedBootstrap; + + try { + const skillContent = readFileSync(bootstrapSkillPath, "utf8"); + const body = stripFrontmatter(skillContent); + cachedBootstrap = `${EXTREMELY_IMPORTANT_MARKER} +${BOOTSTRAP_MARKER} + +You have superpowers. + +The using-superpowers skill content is included below and is already loaded for this Pi session. Follow it now. Do not try to load using-superpowers again. + +${body} + +${piToolMapping()} +`; + return cachedBootstrap; + } catch { + cachedBootstrap = null; + return null; + } +} + +function stripFrontmatter(content: string): string { + const match = content.match(/^---\n[\s\S]*?\n---\n([\s\S]*)$/); + return (match ? match[1] : content).trim(); +} + +function piToolMapping(): string { + return `## Pi tool mapping + +Pi has native skills but does not expose Claude Code's \`Skill\` tool. When a Superpowers instruction says to invoke a skill, use Pi's native skill system instead: load the relevant \`SKILL.md\` with \`read\` when the skill applies, or let a human invoke \`/skill:name\` explicitly. + +Pi's built-in coding tools are lowercase: \`read\`, \`write\`, \`edit\`, \`bash\`, plus optional \`grep\`, \`find\`, and \`ls\`. Use those for the corresponding actions: read a file, create or edit files, run shell commands, search file contents, find files by name, and list directories. + +Pi does not ship a standard subagent tool. If a subagent tool such as \`subagent\` from \`pi-subagents\` is available, use it for Superpowers subagent workflows. If no subagent tool is available, do the work in this session or explain the missing capability instead of inventing \`Task\` calls. + +Pi does not ship a standard task-list tool. If an installed todo/task tool is available, use it. Otherwise track work in plan files or a repo-local \`TODO.md\` when task tracking is needed. Treat older \`TodoWrite\` references as this task-tracking action.`; +} + +function messageContainsBootstrap(message: unknown): boolean { + const content = (message as { content?: unknown }).content; + if (typeof content === "string") return content.includes(BOOTSTRAP_MARKER); + if (!Array.isArray(content)) return false; + return content.some((part) => { + return ( + part && + typeof part === "object" && + (part as { type?: unknown }).type === "text" && + typeof (part as { text?: unknown }).text === "string" && + (part as { text: string }).text.includes(BOOTSTRAP_MARKER) + ); + }); +} + +function firstNonCompactionSummaryIndex(messages: unknown[]): number { + let index = 0; + while ((messages[index] as { role?: unknown } | undefined)?.role === "compactionSummary") { + index += 1; + } + return index; +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..08fd36d126 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +repos: + - repo: local + hooks: + - id: evals-ruff-check + name: evals ruff check + entry: uv --project evals run ruff check + language: system + files: ^evals/.*\.py$ + + - id: evals-ruff-format-check + name: evals ruff format --check + entry: uv --project evals run ruff format --check + language: system + files: ^evals/.*\.py$ + + - id: evals-ty-check + name: evals ty check + entry: uv --directory evals run ty check + language: system + pass_filenames: false + files: ^evals/.*\.py$ diff --git a/CLAUDE.md b/CLAUDE.md index 9bd3e00bd1..ffa56a718a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,6 +94,10 @@ Skills are not prose — they are code that shapes agent behavior. If you modify - Show before/after eval results in your PR - Do not modify carefully-tuned content (Red Flags tables, rationalization lists, "human partner" language) without evidence the change is an improvement +## Eval harness + +Skill-behavior evals live in the `evals/` submodule — after cloning, run `git submodule update --init evals`, then see `evals/README.md`. Drill (the harness) drives real tmux sessions of Claude Code / Codex / Gemini CLI and judges skill compliance with an LLM verifier. Plugin-infrastructure tests still live at `tests/`. + ## Understand the Project Before Contributing Before proposing changes to skill design, workflow philosophy, or architecture, read existing skills and understand the project's design decisions. Superpowers has its own tested philosophy about skill design, agent behavior shaping, and terminology (e.g., "your human partner" is deliberate, not interchangeable with "the user"). Changes that rewrite the project's voice or restructure its approach without understanding why it exists will be rejected. diff --git a/README.md b/README.md index ea17e30e0a..ea7b53aa7a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Superpowers is a complete software development methodology for your coding agent ## Quickstart -Give your agent Superpowers: [Claude Code](#claude-code), [Codex CLI](#codex-cli), [Codex App](#codex-app), [Factory Droid](#factory-droid), [Gemini CLI](#gemini-cli), [OpenCode](#opencode), [Cursor](#cursor), [GitHub Copilot CLI](#github-copilot-cli). +Give your agent Superpowers: [Claude Code](#claude-code), [Codex App](#codex-app), [Codex CLI](#codex-cli), [Cursor](#cursor), [Factory Droid](#factory-droid), [Gemini CLI](#gemini-cli), [GitHub Copilot CLI](#github-copilot-cli), [OpenCode](#opencode), [Pi](#pi). ## How it works @@ -14,7 +14,7 @@ Once it's teased a spec out of the conversation, it shows it to you in chunks sh After you've signed off on the design, your agent puts together an implementation plan that's clear enough for an enthusiastic junior engineer with poor taste, no judgement, no project context, and an aversion to testing to follow. It emphasizes true red/green TDD, YAGNI (You Aren't Gonna Need It), and DRY. -Next up, once you say "go", it launches a *subagent-driven-development* process, having agents work through each engineering task, inspecting and reviewing their work, and continuing forward. It's not uncommon for Claude to be able to work autonomously for a couple hours at a time without deviating from the plan you put together. +Next up, once you say "go", it launches a *subagent-driven-development* process, having agents work through each engineering task, inspecting and reviewing their work, and continuing forward. It's not uncommon for your agent to work autonomously for a couple hours at a time without deviating from the plan you put together. There's a bunch more to it, but that's the core of the system. And because the skills trigger automatically, you don't need to do anything special. Your coding agent just has Superpowers. @@ -25,7 +25,7 @@ If Superpowers has helped you do stuff that makes money and you are so inclined, Thanks! -- Jesse +\- Jesse ## Installation @@ -60,6 +60,14 @@ The Superpowers marketplace provides Superpowers and some other related plugins /plugin install superpowers@superpowers-marketplace ``` +### Codex App + +Superpowers is available via the [official Codex plugin marketplace](https://github.com/openai/plugins). + +- In the Codex app, click on Plugins in the sidebar. +- You should see `Superpowers` in the Coding section. +- Click the `+` next to Superpowers and follow the prompts. + ### Codex CLI Superpowers is available via the [official Codex plugin marketplace](https://github.com/openai/plugins). @@ -78,13 +86,15 @@ Superpowers is available via the [official Codex plugin marketplace](https://git - Select `Install Plugin`. -### Codex App +### Cursor -Superpowers is available via the [official Codex plugin marketplace](https://github.com/openai/plugins). +- In Cursor Agent chat, install from marketplace: -- In the Codex app, click on Plugins in the sidebar. -- You should see `Superpowers` in the Coding section. -- Click the `+` next to Superpowers and follow the prompts. + ```text + /add-plugin superpowers + ``` + +- Or search for "superpowers" in the plugin marketplace. ### Factory Droid @@ -114,6 +124,20 @@ Superpowers is available via the [official Codex plugin marketplace](https://git gemini extensions update superpowers ``` +### GitHub Copilot CLI + +- Register the marketplace: + + ```bash + copilot plugin marketplace add obra/superpowers-marketplace + ``` + +- Install the plugin: + + ```bash + copilot plugin install superpowers@superpowers-marketplace + ``` + ### OpenCode OpenCode uses its own plugin install; install Superpowers separately even if you @@ -127,29 +151,21 @@ already use it in another harness. - Detailed docs: [docs/README.opencode.md](docs/README.opencode.md) -### Cursor - -- In Cursor Agent chat, install from marketplace: - - ```text - /add-plugin superpowers - ``` - -- Or search for "superpowers" in the plugin marketplace. +### Pi -### GitHub Copilot CLI +Install Superpowers as a Pi package from this repository: -- Register the marketplace: +```bash +pi install git:github.com/obra/superpowers +``` - ```bash - copilot plugin marketplace add obra/superpowers-marketplace - ``` +For local development, run Pi with this checkout loaded as a temporary package: -- Install the plugin: +```bash +pi -e /path/to/superpowers +``` - ```bash - copilot plugin install superpowers@superpowers-marketplace - ``` +The Pi package loads the Superpowers skills and a small extension that injects the `using-superpowers` bootstrap at session startup and again after compaction. Pi has native skills, so no compatibility `Skill` tool is required. Subagent and task-list tools remain optional Pi companion packages. ## The Basic Workflow @@ -214,6 +230,8 @@ The general contribution process for Superpowers is below. Keep in mind that we 4. Follow the `writing-skills` skill for creating and testing new and modified skills 5. Submit a PR, being sure to fill in the pull request template. +Skill-behavior tests use the eval harness submodule at `evals/`. After cloning this repo, run `git submodule update --init evals`, then see `evals/README.md` for setup. Plugin-infrastructure tests live at `tests/` and run via the relevant `run-*.sh` or `npm test`. + See `skills/writing-skills/SKILL.md` for the complete guide. ## Updating diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 50944a692c..8f196513fd 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -50,6 +50,8 @@ New `sync-to-codex-plugin` script mirrors superpowers into the OpenAI Codex plug - **Single source of truth** — the persona/checklist that previously lived in both `agents/code-reviewer.md` and the skill's placeholder template (and drifted independently) is now one file. - **`subagent-driven-development` follows suit** — its `code-quality-reviewer-prompt.md` now dispatches `Task (general-purpose)` instead of the named agent. - **Behavioral test added** — `tests/claude-code/test-requesting-code-review.sh` plants real bugs (SQL injection, plaintext password handling, credential logging) into a tiny project and asserts the dispatched reviewer flags every planted issue at Critical/Important severity and refuses to approve the diff. + +> Note: `tests/claude-code/test-requesting-code-review.sh` and `tests/claude-code/test-document-review-system.sh` (mentioned later in this document) were lifted into drill scenarios on 2026-05-06 and removed from `tests/`. See `evals/scenarios/code-review-catches-planted-bugs.yaml` and `evals/scenarios/spec-reviewer-catches-planted-flaws.yaml`. The references above and below are preserved as dated artifacts of the work this section describes. - **Codex and Copilot workaround docs trimmed** — the "Named agent dispatch" sections in `references/codex-tools.md` and `references/copilot-tools.md` documented how to flatten a named agent into a generic dispatch. With no named agents shipping, the workaround is unnecessary; both sections were dropped. ### Subagent-Driven Development diff --git a/docs/README.opencode.md b/docs/README.opencode.md index 951206692a..11da85425a 100644 --- a/docs/README.opencode.md +++ b/docs/README.opencode.md @@ -50,7 +50,7 @@ use skill tool to list skills ### Loading a Skill ``` -use skill tool to load superpowers/brainstorming +use skill tool to load brainstorming ``` ### Personal Skills @@ -99,17 +99,23 @@ To pin a specific version, use a branch or tag: The plugin does two things: -1. **Injects bootstrap context** via the `experimental.chat.system.transform` hook, adding superpowers awareness to every conversation. +1. **Injects bootstrap context** via the `experimental.chat.messages.transform` hook, adding superpowers awareness to every conversation. 2. **Registers the skills directory** via the `config` hook, so OpenCode discovers all superpowers skills without symlinks or manual config. ### Tool Mapping -Skills written for Claude Code are automatically adapted for OpenCode: +Skills speak in actions rather than naming any one runtime's tools. On OpenCode these resolve to: -- `TodoWrite` → `todowrite` -- `Task` with subagents → OpenCode's `@mention` system -- `Skill` tool → OpenCode's native `skill` tool -- File operations → Native OpenCode tools +- "Create a todo" / "mark complete in todo list" → `todowrite` +- `Subagent (general-purpose):` template → OpenCode's `task` tool with `subagent_type: "general"` (or `"explore"` for codebase exploration) +- "Invoke a skill" → OpenCode's native `skill` tool +- "Read a file" → `read` +- "Create a file" / "edit a file" / "delete a file" → `apply_patch` +- "Run a shell command" → `bash` +- "Search file contents" / "find files by name" → `grep`, `glob` +- "Fetch a URL" → `webfetch` + +(Verified against the installed OpenCode CLI's tool inventory.) ## Troubleshooting @@ -147,7 +153,7 @@ Then use the installed package path in `opencode.json`: ### Bootstrap not appearing -1. Check OpenCode version supports `experimental.chat.system.transform` hook +1. Check OpenCode version supports `experimental.chat.messages.transform` hook 2. Restart OpenCode after config changes ## Getting Help diff --git a/docs/superpowers/plans/2026-03-23-codex-app-compatibility.md b/docs/superpowers/plans/2026-03-23-codex-app-compatibility.md index 933cddfd60..0f35d3d283 100644 --- a/docs/superpowers/plans/2026-03-23-codex-app-compatibility.md +++ b/docs/superpowers/plans/2026-03-23-codex-app-compatibility.md @@ -555,6 +555,8 @@ Should show exactly 6 files changed (5 skill files + 1 test file). No other file If test runner exists: ```bash # Run skill-triggering tests +# Note: tests/skill-triggering/ was lifted into drill scenarios on 2026-05-06. +# See evals/scenarios/triggering-*.yaml. The reference below is a dated artifact. ./tests/skill-triggering/run-all.sh 2>/dev/null || echo "Skill triggering tests not available in this environment" # Run SDD integration test diff --git a/docs/superpowers/plans/2026-04-06-worktree-rototill.md b/docs/superpowers/plans/2026-04-06-worktree-rototill.md index ae5acec199..183ee744d9 100644 --- a/docs/superpowers/plans/2026-04-06-worktree-rototill.md +++ b/docs/superpowers/plans/2026-04-06-worktree-rototill.md @@ -275,23 +275,16 @@ If no native tool is available, create a worktree manually using git. Follow this priority order: -1. **Check existing directories:** +1. **Check your instructions for a worktree directory preference.** If specified, use it without asking. + +2. **Check existing project-local directories:** ```bash ls -d .worktrees 2>/dev/null # Preferred (hidden) ls -d worktrees 2>/dev/null # Alternative ``` If found, use that directory. If both exist, `.worktrees` wins. -2. **Check for existing global directory:** - ```bash - project=$(basename "$(git rev-parse --show-toplevel)") - ls -d ~/.config/superpowers/worktrees/$project 2>/dev/null - ``` - If found, use it (backward compatibility with legacy global path). - -3. **Check your instructions for a worktree directory preference.** If specified, use it without asking. - -4. **Default to `.worktrees/`.** +3. **Default to `.worktrees/`.** #### Safety Verification (project-local directories only) @@ -305,16 +298,11 @@ git check-ignore -q .worktrees 2>/dev/null || git check-ignore -q worktrees 2>/d **Why critical:** Prevents accidentally committing worktree contents to repository. -Global directories (`~/.config/superpowers/worktrees/`) need no verification. - #### Create the Worktree ```bash -project=$(basename "$(git rev-parse --show-toplevel)") - # Determine path based on chosen location -# For project-local: path="$LOCATION/$BRANCH_NAME" -# For global: path="~/.config/superpowers/worktrees/$project/$BRANCH_NAME" +path="$LOCATION/$BRANCH_NAME" git worktree add "$path" -b "$BRANCH_NAME" cd "$path" @@ -387,7 +375,6 @@ Ready to implement | `worktrees/` exists | Use it (verify ignored) | | Both exist | Use `.worktrees/` | | Neither exists | Check instruction file, then default `.worktrees/` | -| Global path exists | Use it (backward compat) | | Directory not ignored | Add to .gitignore + commit | | Permission error on create | Sandbox fallback, work in place | | Tests fail during baseline | Report failures + ask | @@ -464,7 +451,7 @@ git commit -m "feat: rewrite using-git-worktrees with detect-and-defer (PRI-974) Step 0: GIT_DIR != GIT_COMMON detection (skip if already isolated) Step 0 consent: opt-in prompt before creating worktree (#991) Step 1a: native tool preference (short, first, declarative) -Step 1b: git worktree fallback with hooks symlink and legacy path compat +Step 1b: git worktree fallback with project-local directory policy Submodule guard prevents false detection Platform-neutral instruction file references (#1049)" ``` @@ -663,7 +650,7 @@ WORKTREE_PATH=$(git rev-parse --show-toplevel) **If `GIT_DIR == GIT_COMMON`:** Normal repo, no worktree to clean up. Done. -**If worktree path is under `.worktrees/` or `~/.config/superpowers/worktrees/`:** Superpowers created this worktree — we own cleanup. +**If worktree path is under `.worktrees/` or `worktrees/`:** Superpowers created this worktree — we own cleanup. ```bash MAIN_ROOT=$(git -C "$(git rev-parse --git-common-dir)/.." rev-parse --show-toplevel) @@ -707,7 +694,7 @@ git worktree prune # Self-healing: clean up any stale registrations **Cleaning up harness-owned worktrees** - **Problem:** Removing a worktree the harness created causes phantom state -- **Fix:** Only clean up worktrees under `.worktrees/` or `~/.config/superpowers/worktrees/` +- **Fix:** Only clean up worktrees under `.worktrees/` or `worktrees/` **No confirmation for discard** - **Problem:** Accidentally delete work diff --git a/docs/superpowers/plans/2026-05-06-lift-drill-into-evals.md b/docs/superpowers/plans/2026-05-06-lift-drill-into-evals.md new file mode 100644 index 0000000000..b1c01ca26e --- /dev/null +++ b/docs/superpowers/plans/2026-05-06-lift-drill-into-evals.md @@ -0,0 +1,1374 @@ +# Lift drill into superpowers as `evals/` — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Move the standalone `obra/drill` skill-compliance benchmark into superpowers as a top-level `evals/` directory, delete redundant bash tests under `superpowers/tests/` after per-file subagent verification of drill scenario coverage, and update top-level docs so contributors land on the new structure. + +**Architecture:** Single PR against `dev` on a new branch `f/evals-lift`. Drill source is copied verbatim with explicit rsync excludes to keep `.git/`, `.venv/`, etc. out of the new dir. A small helper in `drill/cli.py` defaults `SUPERPOWERS_ROOT` to the parent of the `evals/` directory, so contributors don't have to set the env var. Each bash-test deletion is gated by a subagent that compares the bash test's assertions to its claimed drill scenario's verify block. Historical references in plan docs and release notes are annotated, not rewritten. + +**Tech Stack:** Python 3.11 + uv (drill's existing toolchain, unchanged); rsync; bash; git. + +**Spec:** `docs/superpowers/specs/2026-05-06-lift-drill-into-evals-design.md` — read this first. + +**Drill source location:** `/Users/jesse/Documents/GitHub/superpowers/drill/` (sibling to `superpowers/`). + +--- + +## Task 1: Branch off dev + +**Files:** none (git operation only) + +- [ ] **Step 1: Verify clean working tree** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git status --short +``` + +Expected: empty output (or only untracked `.opencode/package-lock.json`, which is fine). + +- [ ] **Step 2: Fetch latest dev** + +```bash +git fetch origin dev:dev +``` + +- [ ] **Step 3: Create the branch** + +```bash +git checkout -b f/evals-lift dev +``` + +Expected: `Switched to a new branch 'f/evals-lift'`. + +- [ ] **Step 4: Sanity check** + +```bash +git log --oneline -1 +``` + +Expected output begins with whatever commit `origin/dev` points to (currently `b4363df docs: turned the dash in "- Jesse" into an escape sequence (#1474)`). + +--- + +## Task 2: Capture drill SHA at copy time + +**Files:** none (records the value for the lift commit message) + +- [ ] **Step 1: Get the current drill HEAD SHA** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/drill +DRILL_SHA=$(git rev-parse HEAD) +echo "$DRILL_SHA" +``` + +- [ ] **Step 2: Verify drill has no uncommitted work** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/drill +git status --short +``` + +Expected: empty (no untracked or modified files). If output is non-empty, stop and report — drill working tree must be clean before lift, otherwise the SHA-pin is meaningless. + +- [ ] **Step 3: Save the SHA in shell env for next task** + +```bash +echo "DRILL_SHA=$DRILL_SHA" # write this down for use in Task 3 +``` + +--- + +## Task 3: rsync drill into evals/ + +**Files:** +- Create: `evals/` (entire directory tree from drill, minus excludes) + +- [ ] **Step 1: Verify source and destination paths** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +test -d /Users/jesse/Documents/GitHub/superpowers/drill && echo "drill source: OK" +test ! -d evals && echo "evals/ does not yet exist: OK" +``` + +Expected: both echoes print. + +- [ ] **Step 2: rsync drill to evals/ with explicit excludes** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +rsync -a \ + --exclude=.git \ + --exclude=.venv \ + --exclude=results \ + --exclude=.env \ + --exclude=__pycache__ \ + --exclude='*.egg-info' \ + --exclude=.private-journal \ + --exclude='*.pyc' \ + /Users/jesse/Documents/GitHub/superpowers/drill/ \ + evals/ +``` + +- [ ] **Step 3: Verify excludes worked** + +```bash +find evals -name '.git' -type d +find evals -name '.venv' -type d +find evals -name 'results' -type d +find evals -name '.env' +find evals -name '__pycache__' -type d +find evals -name '*.egg-info' -type d +``` + +Expected: every command returns no output. If any returns a path, manually `rm -rf` it before continuing. + +- [ ] **Step 4: Confirm the source SHA for the commit message** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/drill +DRILL_SHA=$(git rev-parse HEAD) +echo "$DRILL_SHA" +``` + +Expected: the SHA from Task 2 step 1. + +- [ ] **Step 5: Stage everything** + +```bash +git add evals/ +git status --short | head -20 +``` + +Expected output starts with `A evals/...` lines listing many added files. Many of these are in scenarios/, drill/, backends/, setup_helpers/, etc. + +- [ ] **Step 6: Commit** + +```bash +: "${DRILL_SHA:?Set DRILL_SHA from Task 2 before committing}" +git commit -m "$(cat < /tmp/drill-files.txt +wc -l /tmp/drill-files.txt +``` + +- [ ] **Step 2: Get list of files in evals/** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +find evals -type f | sed 's|^evals/|./|' | sort > /tmp/evals-files.txt +wc -l /tmp/evals-files.txt +``` + +- [ ] **Step 3: Diff the two lists** + +The file lists should match exactly after excluded paths are removed. + +```bash +diff /tmp/drill-files.txt /tmp/evals-files.txt +``` + +Expected: no output. + +- [ ] **Step 4: Per-file checksum verification** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/drill +while read -r f; do + sha1=$(shasum -a 256 "$f" | cut -d' ' -f1) + sha2=$(shasum -a 256 "/Users/jesse/Documents/GitHub/superpowers/superpowers/evals/${f#./}" | cut -d' ' -f1) + if [ "$sha1" != "$sha2" ]; then + echo "MISMATCH: $f ($sha1 vs $sha2)" + fi +done < /tmp/drill-files.txt | head -20 +``` + +Expected: no output (every file's checksum matches between drill and evals). + +- [ ] **Step 5: Smoke check - install dependencies** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +uv sync +``` + +Expected: `Installed N packages` or similar. No errors. + +- [ ] **Step 6: Smoke check - drill list** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +uv run drill list 2>&1 | head -5 +``` + +Expected: starts with scenario names. (Will likely error or warn about missing SUPERPOWERS_ROOT — that's fine, fixed in next task.) + +- [ ] **Step 7: Dispatch verification subagent** + +Dispatch a `general-purpose` subagent with this prompt: + +``` +You are verifying a verbatim copy of the drill repo at +/Users/jesse/Documents/GitHub/superpowers/drill into +/Users/jesse/Documents/GitHub/superpowers/superpowers/evals. + +Verify: + +1. The lift commit message records the SHA reported by: + cd /Users/jesse/Documents/GitHub/superpowers/drill && git rev-parse HEAD + +2. None of these excluded paths exist under evals/: .git/, .venv/, +results/, .env/, __pycache__/, *.egg-info/, .private-journal/. + +3. Every non-excluded file in drill has a SHA-256-identical +counterpart in evals/, and there are no extra files in evals/. + +4. The pyproject.toml, uv.lock, scenarios/*.yaml, backends/*.yaml, +setup_helpers/*.py, drill/*.py, prompts/*.md, fixtures/, bin/, and +docs/ are all present. + +Report each check with PASS/FAIL. If any FAIL, dump enough detail +that the parent can fix. +``` + +If the subagent reports any FAIL, fix the underlying issue (delete the leaked file, re-rsync, etc.) before continuing. + +--- + +## Task 5: Add `SUPERPOWERS_ROOT` default helper + +**Files:** +- Modify: `evals/drill/cli.py:11-14` + +- [ ] **Step 1: Read the current cli.py header** + +```bash +sed -n '1,20p' /Users/jesse/Documents/GitHub/superpowers/superpowers/evals/drill/cli.py +``` + +Expected output: + +```python +"""Drill CLI: run, compare, list.""" + +from __future__ import annotations + +import secrets +from pathlib import Path + +import click +from dotenv import load_dotenv + +PROJECT_ROOT: Path = Path(__file__).parent.parent + +load_dotenv(PROJECT_ROOT / ".env") +``` + +- [ ] **Step 2: Write a failing test for the helper** + +Open `evals/tests/test_cli.py` and add this test at the end: + +```python +def test_set_superpowers_root_default_when_unset(monkeypatch, tmp_path): + """When SUPERPOWERS_ROOT is unset, helper sets it to PROJECT_ROOT.parent.""" + monkeypatch.delenv("SUPERPOWERS_ROOT", raising=False) + from drill.cli import _set_superpowers_root_default, PROJECT_ROOT + + _set_superpowers_root_default() + + import os + assert os.environ["SUPERPOWERS_ROOT"] == str(PROJECT_ROOT.parent) + + +def test_set_superpowers_root_default_respects_existing(monkeypatch): + """When SUPERPOWERS_ROOT is already set, helper does not override.""" + monkeypatch.setenv("SUPERPOWERS_ROOT", "/custom/path") + from drill.cli import _set_superpowers_root_default + + _set_superpowers_root_default() + + import os + assert os.environ["SUPERPOWERS_ROOT"] == "/custom/path" +``` + +- [ ] **Step 3: Run the test and watch it fail** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +uv run pytest tests/test_cli.py -k set_superpowers_root_default -v +``` + +Expected: 2 tests fail with `AttributeError: module 'drill.cli' has no attribute '_set_superpowers_root_default'`. + +- [ ] **Step 4: Add the helper to cli.py** + +Edit `/Users/jesse/Documents/GitHub/superpowers/superpowers/evals/drill/cli.py`. Replace lines 1–14 with: + +```python +"""Drill CLI: run, compare, list.""" + +from __future__ import annotations + +import os +import secrets +from pathlib import Path + +import click +from dotenv import load_dotenv + +PROJECT_ROOT: Path = Path(__file__).parent.parent + +load_dotenv(PROJECT_ROOT / ".env") + + +def _set_superpowers_root_default() -> None: + """Default SUPERPOWERS_ROOT to the parent of evals/ if not already set. + + Drill historically required contributors to export SUPERPOWERS_ROOT + pointing at the superpowers checkout. After lifting drill into + superpowers/evals/, the parent of PROJECT_ROOT is always the + superpowers root, so we can supply this default automatically. + + Existing SUPERPOWERS_ROOT environment values are respected as overrides. + """ + os.environ.setdefault("SUPERPOWERS_ROOT", str(PROJECT_ROOT.parent)) + + +_set_superpowers_root_default() +``` + +The bottom-of-module call to `_set_superpowers_root_default()` runs at import time, immediately after `load_dotenv()`. This ensures both `engine.py` and `setup.py` (which read `os.environ["SUPERPOWERS_ROOT"]` directly) and the YAML interpolation (which reads `os.environ` when the backend YAML is loaded) all see the value. + +- [ ] **Step 5: Run the test and watch it pass** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +uv run pytest tests/test_cli.py -k set_superpowers_root_default -v +``` + +Expected: 2 tests pass. + +- [ ] **Step 6: Commit** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git add evals/drill/cli.py evals/tests/test_cli.py +git commit -m "evals: default SUPERPOWERS_ROOT to parent of evals/ if unset + +Adds _set_superpowers_root_default() to drill/cli.py, called at +module import after load_dotenv(). PROJECT_ROOT resolves to evals/ +post-lift; its parent is the superpowers repo root, which is the +correct value for SUPERPOWERS_ROOT. + +Existing env values are respected as overrides via os.environ.setdefault. + +Tests: +- helper sets default when var is unset +- helper does not override when var is already set" +``` + +--- + +## Task 6: Update backend YAMLs to reflect the new env contract + +**Files:** +- Modify: `evals/backends/codex.yaml` (drop `SUPERPOWERS_ROOT` from `required_env`) +- Modify: `evals/backends/gemini.yaml` (drop `SUPERPOWERS_ROOT` from `required_env`) + +The five `claude*.yaml` backend configs interpolate `${SUPERPOWERS_ROOT}` into `args` for the `--plugin-dir` flag — they keep `SUPERPOWERS_ROOT` in `required_env` because the interpolation needs it. The codex/gemini configs only listed it for engine.py/setup.py's `os.environ` reads, which the helper now satisfies. + +- [ ] **Step 1: Confirm current state** + +```bash +grep -A3 'required_env:' /Users/jesse/Documents/GitHub/superpowers/superpowers/evals/backends/codex.yaml +grep -A2 'required_env:' /Users/jesse/Documents/GitHub/superpowers/superpowers/evals/backends/gemini.yaml +``` + +Expected outputs include `- SUPERPOWERS_ROOT` lines. + +- [ ] **Step 2: Read codex.yaml fully** + +```bash +cat /Users/jesse/Documents/GitHub/superpowers/superpowers/evals/backends/codex.yaml +``` + +- [ ] **Step 3: Edit codex.yaml — drop the `- SUPERPOWERS_ROOT` line under `required_env`** + +Open `evals/backends/codex.yaml` and find: + +```yaml +required_env: + - OPENAI_API_KEY + - SUPERPOWERS_ROOT +``` + +Replace with: + +```yaml +required_env: + - OPENAI_API_KEY +``` + +- [ ] **Step 4: Edit gemini.yaml — drop the `- SUPERPOWERS_ROOT` line under `required_env`** + +Open `evals/backends/gemini.yaml` and find: + +```yaml +required_env: + - SUPERPOWERS_ROOT +``` + +Replace with: + +```yaml +required_env: [] +``` + +(Empty list rather than dropping the field, so YAML schema validation doesn't trip.) + +- [ ] **Step 5: Run drill's pytest suite to ensure nothing broke** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +uv run pytest -x 2>&1 | tail -20 +``` + +Expected: all tests pass. If `tests/test_backend.py` complains about `required_env` membership for codex/gemini, see Task 7. + +- [ ] **Step 6: Commit** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git add evals/backends/codex.yaml evals/backends/gemini.yaml +git commit -m "evals: drop SUPERPOWERS_ROOT from codex/gemini required_env + +These backends only read SUPERPOWERS_ROOT via engine.py/setup.py's +os.environ access, which the new cli.py default helper supplies +automatically. claude*.yaml keep SUPERPOWERS_ROOT in required_env +because they interpolate \${SUPERPOWERS_ROOT} into --plugin-dir args." +``` + +--- + +## Task 7: Update drill's pytest suite for the new contract + +**Files:** +- Modify: `evals/tests/test_backend.py` (per-test updates if Task 6 step 5 surfaced failures) + +- [ ] **Step 1: Run the test suite** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +uv run pytest tests/test_backend.py -v 2>&1 | tail -30 +``` + +If all tests pass, skip to step 5 (commit nothing, move to Task 8). Otherwise: + +- [ ] **Step 2: Read failing tests** + +For each failure, open the test in `evals/tests/test_backend.py` and read the assertion. + +- [ ] **Step 3: Update assertions** + +For tests that assert `SUPERPOWERS_ROOT` membership in `codex.yaml`'s or `gemini.yaml`'s `required_env`: invert the assertion to confirm absence. Example: + +```python +# Before: +def test_codex_requires_superpowers_root(): + backend = load_backend("codex") + assert "SUPERPOWERS_ROOT" in backend.required_env + +# After: +def test_codex_does_not_require_superpowers_root(): + """codex.yaml dropped SUPERPOWERS_ROOT from required_env; + the cli.py helper supplies the default.""" + backend = load_backend("codex") + assert "SUPERPOWERS_ROOT" not in backend.required_env +``` + +- [ ] **Step 4: Re-run the test suite** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +uv run pytest -x 2>&1 | tail -10 +``` + +Expected: all tests pass. + +- [ ] **Step 5: Commit (only if step 1 had failures)** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git add evals/tests/test_backend.py +git commit -m "evals: update test_backend.py for relaxed required_env contract" +``` + +--- + +## Task 8: Update evals/README.md and evals/CLAUDE.md + +**Files:** +- Modify: `evals/README.md` (drop SUPERPOWERS_ROOT setup step) +- Modify: `evals/CLAUDE.md` (drop SUPERPOWERS_ROOT setup step) + +- [ ] **Step 1: Edit evals/README.md** + +Find the section that looks like: + +```markdown +Required environment: +```bash +export SUPERPOWERS_ROOT=/path/to/superpowers +export ANTHROPIC_API_KEY=sk-... +``` +``` + +Replace with: + +```markdown +Required environment: +```bash +export ANTHROPIC_API_KEY=sk-... +``` + +`SUPERPOWERS_ROOT` defaults to the parent of `evals/` (the superpowers repo root) and only needs to be set if you're running drill against a different superpowers checkout. +``` + +- [ ] **Step 2: Edit evals/CLAUDE.md** + +Find the section: + +```markdown +## Required env + +``` +SUPERPOWERS_ROOT=/path/to/superpowers +ANTHROPIC_API_KEY=sk-... +``` +``` + +Replace with: + +```markdown +## Required env + +``` +ANTHROPIC_API_KEY=sk-... +``` + +`SUPERPOWERS_ROOT` defaults to the parent of `evals/` (the superpowers repo root). Override only if running drill against a different superpowers checkout. +``` + +- [ ] **Step 3: Commit** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git add evals/README.md evals/CLAUDE.md +git commit -m "evals: drop SUPERPOWERS_ROOT setup step from README/CLAUDE + +The cli.py helper now defaults the env var. Mention as override only." +``` + +--- + +## Task 9: Validate from new location + +**Files:** none (validation only — no commit unless something needs fixing) + +- [ ] **Step 1: Run drill's full pytest suite** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +unset SUPERPOWERS_ROOT +uv run pytest 2>&1 | tail -5 +``` + +Expected: all tests pass. The `unset` ensures we're testing the helper, not an inherited env var. + +- [ ] **Step 2: Run drill list** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +unset SUPERPOWERS_ROOT +uv run drill list 2>&1 | head -10 +``` + +Expected: scenario list, no error about missing SUPERPOWERS_ROOT. + +- [ ] **Step 3: Source the env file** + +```bash +set -a +source /Users/jesse/Documents/GitHub/prime-radiant-inc/sprout/.env +set +a +echo "ANTHROPIC_API_KEY set: ${ANTHROPIC_API_KEY:+yes}" +``` + +Expected: `ANTHROPIC_API_KEY set: yes`. + +- [ ] **Step 4: Run a cheap drill scenario** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +unset SUPERPOWERS_ROOT +uv run drill run triggering-test-driven-development -b claude 2>&1 | tail -3 +``` + +Expected: `claude: 1 passed, 0 failed, 0 errors`. + +If FAIL, debug before continuing. The path-defaults change is the most likely culprit; check that the helper actually fired by adding a `print(os.environ["SUPERPOWERS_ROOT"])` after the helper call temporarily. + +--- + +## Task 10: Bash test deletion phase — per-file with subagent gate + +This task has many sub-steps because each candidate-deletion file gets its own subagent verification + commit. The candidate list comes from the spec's coverage map. For each entry below: + +1. Read the bash test file. +2. Read the candidate drill scenario YAML. +3. Dispatch a subagent with both contents and the comparison prompt. +4. Subagent reports per-assertion match table. +5. If every bash assertion has a match: delete the bash test, commit. +6. If any unmatched: stop, escalate, do not delete. + +**Subagent prompt template (use for every deletion):** + +``` +You are gating a bash test deletion. The bash test is allegedly +covered by a drill scenario; your job is to verify that claim. + +BASH TEST: + +DRILL SCENARIO: + +Output a markdown table with columns: BASH ASSERTION, DRILL CHECK, +STATUS. List EVERY assertion the bash test makes (every grep, every +[ ], every test command, every PASS/FAIL emit). For each, find a +matching drill check (in verify.assertions or verify.criteria) or +mark as UNMATCHED. + +After the table, output "VERDICT: SAFE TO DELETE" if every bash +assertion has a match, otherwise "VERDICT: KEEP — N unmatched +assertions". Be conservative: if you are uncertain about a match, +mark as UNMATCHED. +``` + +### Task 10a: Skill-triggering prompts (6 files) + +**Files:** +- Delete: `tests/skill-triggering/prompts/dispatching-parallel-agents.txt` +- Delete: `tests/skill-triggering/prompts/executing-plans.txt` +- Delete: `tests/skill-triggering/prompts/requesting-code-review.txt` +- Delete: `tests/skill-triggering/prompts/systematic-debugging.txt` +- Delete: `tests/skill-triggering/prompts/test-driven-development.txt` +- Delete: `tests/skill-triggering/prompts/writing-plans.txt` +- Keep: `tests/skill-triggering/run-test.sh`, `run-all.sh` + +These prompt files are inputs to the bash runner — they don't have their own assertions. The runner script does the assertion. Map each prompt to its drill scenario: + +| Prompt | Drill scenario | +|--------|----------------| +| dispatching-parallel-agents.txt | triggering-dispatching-parallel-agents.yaml | +| executing-plans.txt | triggering-executing-plans.yaml | +| requesting-code-review.txt | triggering-requesting-code-review.yaml | +| systematic-debugging.txt | triggering-systematic-debugging.yaml | +| test-driven-development.txt | triggering-test-driven-development.yaml | +| writing-plans.txt | triggering-writing-plans.yaml | + +- [ ] **Step 1: For each prompt file, dispatch the subagent** + +For prompt `tests/skill-triggering/prompts/.txt` and scenario `evals/scenarios/triggering-.yaml`, run the subagent prompt template with both contents pasted in. The subagent's job is to verify the prompt content matches what the drill scenario's `turns[].intent` describes. + +If all 6 verify SAFE TO DELETE, proceed to step 2. If any verifies KEEP, that one stays and the rest may still proceed. + +- [ ] **Step 2: Verify the runner is still useful for unrelated cases** + +```bash +ls /Users/jesse/Documents/GitHub/superpowers/superpowers/tests/skill-triggering/prompts/ +``` + +If the prompts/ directory is empty after the planned deletions, also delete `tests/skill-triggering/run-test.sh` and `run-all.sh` (they have nothing to run). Otherwise keep the runner. + +- [ ] **Step 3: Delete and commit** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git rm tests/skill-triggering/prompts/dispatching-parallel-agents.txt +git rm tests/skill-triggering/prompts/executing-plans.txt +git rm tests/skill-triggering/prompts/requesting-code-review.txt +git rm tests/skill-triggering/prompts/systematic-debugging.txt +git rm tests/skill-triggering/prompts/test-driven-development.txt +git rm tests/skill-triggering/prompts/writing-plans.txt +# If runner is now orphaned: +git rm tests/skill-triggering/run-test.sh tests/skill-triggering/run-all.sh +rmdir tests/skill-triggering/prompts/ 2>/dev/null || true +rmdir tests/skill-triggering/ 2>/dev/null || true +git commit -m "tests: remove skill-triggering bash prompts (covered by drill triggering-* scenarios) + +Subagent verification confirmed each prompt's intent matches its +corresponding drill scenario's turns[].intent. Drill scenarios are +canonical; bash runner has no remaining prompts to drive." +``` + +### Task 10b: explicit-skill-requests (selective deletion) + +**Files:** +- Inspect: 6 files in `tests/explicit-skill-requests/` +- Delete: only those verified to be 100% covered by drill scenarios +- Keep: the rest + +Per the spec's updated coverage map, most of these have no drill counterpart. The likely-deletable ones: + +| Bash test | Candidate drill scenario | Likely outcome | +|-----------|--------------------------|----------------| +| `run-test.sh` | n/a (runner) | KEEP | +| `run-all.sh` | n/a (runner) | KEEP | +| `run-claude-describes-sdd.sh` | `mid-conversation-skill-invocation.yaml` | likely DELETE; verify | +| `run-haiku-test.sh` | none (Haiku-specific) | KEEP | +| `run-multiturn-test.sh`, `run-extended-multiturn-test.sh` | none | KEEP | +| `prompts/please-use-brainstorming.txt`, `prompts/use-systematic-debugging.txt` | none | KEEP | + +- [ ] **Step 1: Read each .sh file and prompt to confirm** + +```bash +for f in /Users/jesse/Documents/GitHub/superpowers/superpowers/tests/explicit-skill-requests/*.sh /Users/jesse/Documents/GitHub/superpowers/superpowers/tests/explicit-skill-requests/prompts/*.txt; do + echo "=== $f ===" + cat "$f" | head -30 +done +``` + +- [ ] **Step 2: Dispatch subagent for `run-claude-describes-sdd.sh` only** + +Use the subagent prompt template above with: +- Bash test content: `tests/explicit-skill-requests/run-claude-describes-sdd.sh` +- Drill scenario: `evals/scenarios/mid-conversation-skill-invocation.yaml` + +- [ ] **Step 3: Act on subagent verdict** + +If SAFE TO DELETE: + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git rm tests/explicit-skill-requests/run-claude-describes-sdd.sh +git commit -m "tests: remove run-claude-describes-sdd.sh (covered by drill mid-conversation-skill-invocation) + +Subagent verification: every assertion matches a drill check. +Other tests in tests/explicit-skill-requests/ are preserved +(run-haiku-test.sh, run-*-multiturn-test.sh, please-use-brainstorming +and use-systematic-debugging prompts have no drill coverage)." +``` + +If KEEP: skip the deletion, document the gap as a future drill-scenario authoring task. + +### Task 10c: subagent-driven-dev real-project tests + +**Files:** +- Inspect: `tests/subagent-driven-dev/go-fractals/`, `tests/subagent-driven-dev/svelte-todo/` +- Candidate scenarios: `evals/scenarios/sdd-go-fractals.yaml`, `evals/scenarios/sdd-svelte-todo.yaml` + +These are entire fixture directories with `design.md`, `plan.md`, `scaffold.sh`. Each fixture directory was lifted into drill as a fixture under `evals/fixtures/`. + +- [ ] **Step 1: Confirm drill has fixture parity** + +```bash +ls /Users/jesse/Documents/GitHub/superpowers/superpowers/evals/fixtures/sdd-go-fractals/ +ls /Users/jesse/Documents/GitHub/superpowers/superpowers/evals/fixtures/sdd-svelte-todo/ +``` + +Expected: each contains `design.md`, `plan.md`, `scaffold.sh` (or equivalent) matching the source under `tests/subagent-driven-dev/`. + +- [ ] **Step 2: Dispatch subagent for each pair** + +Subagent prompt: same template, with bash "test" being the directory's `scaffold.sh` and (if present) any `*.sh` runner. Drill scenario being the corresponding `sdd-*.yaml`. + +- [ ] **Step 3: Act on verdicts** + +For each that returns SAFE TO DELETE: + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git rm -r tests/subagent-driven-dev/go-fractals/ # or svelte-todo +git commit -m "tests: remove subagent-driven-dev/ (covered by drill sdd-) + +Subagent verification: drill scenario asserts test suite passes +post-execution. Fixture content lives at evals/fixtures/sdd-/." +``` + +If both directories are removed, also `git rm -r tests/subagent-driven-dev/` if it becomes empty. + +### Task 10d: tests/claude-code/test-document-review-system.sh + +**Candidate scenario:** `evals/scenarios/spec-reviewer-catches-planted-flaws.yaml` + +- [ ] **Step 1: Dispatch subagent** + +Subagent prompt template with the bash test content and the drill scenario YAML. + +- [ ] **Step 2: Act on verdict** + +If SAFE TO DELETE: + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git rm tests/claude-code/test-document-review-system.sh +git commit -m "tests: remove test-document-review-system.sh (covered by drill spec-reviewer-catches-planted-flaws) + +Subagent verification: every assertion matches a drill check." +``` + +### Task 10e: tests/claude-code/test-requesting-code-review.sh + +**Candidate scenario:** `evals/scenarios/code-review-catches-planted-bugs.yaml` + +- [ ] **Step 1: Dispatch subagent** + +Subagent prompt template with both contents. + +- [ ] **Step 2: Act on verdict** + +If SAFE TO DELETE: + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git rm tests/claude-code/test-requesting-code-review.sh +git commit -m "tests: remove test-requesting-code-review.sh (covered by drill code-review-catches-planted-bugs) + +Subagent verification: every assertion matches a drill check." +``` + +### Task 10f: tests/claude-code/test-worktree-native-preference.sh + +**Candidate scenario:** `evals/scenarios/worktree-creation-under-pressure.yaml` + +- [ ] **Step 1: Dispatch subagent** + +Subagent prompt template with both contents. + +- [ ] **Step 2: Act on verdict** + +If SAFE TO DELETE: + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git rm tests/claude-code/test-worktree-native-preference.sh +git commit -m "tests: remove test-worktree-native-preference.sh (covered by drill worktree-creation-under-pressure) + +Subagent verification: every assertion matches a drill check." +``` + +### Task 10g: tests/claude-code/test-subagent-driven-development-integration.sh + +**Candidate scenario:** `evals/scenarios/sdd-rejects-extra-features.yaml` (partial) + +The spec marks this as "almost certainly keep + extend drill scenario". Don't delete. Instead: + +- [ ] **Step 1: Dispatch subagent for the comparison anyway** + +This documents the gap explicitly. + +- [ ] **Step 2: Decide based on subagent output** + +Likely outcome: KEEP with documented gap. The bash test asserts: `commit_count >= 3`, `npm test` passes, runs `analyze-token-usage.py`. The drill scenario asserts forbidden-exports + reviewer-as-gate. These are mostly disjoint. + +- [ ] **Step 3: Document the gap** (if KEEP) + +Add a comment at the top of `tests/claude-code/test-subagent-driven-development-integration.sh`: + +```bash +# Drill coverage: sdd-rejects-extra-features.yaml covers the YAGNI +# enforcement (forbidden exports + reviewer-as-gate). This bash test +# additionally asserts: ≥3 task commits, npm test passes, token +# analysis runs. Keep until those assertions are added to drill or +# explicitly retired. +``` + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git add tests/claude-code/test-subagent-driven-development-integration.sh +git commit -m "tests: annotate SDD integration test with drill coverage notes + +Drill scenario sdd-rejects-extra-features covers the YAGNI subset. +This bash test adds: ≥3 commits, npm test, token analysis. Kept +until drill scenario covers those or they're retired." +``` + +### Task 10h: tests/claude-code/test-subagent-driven-development.sh + +This is a meta/describe-skill test (per spec). No drill scenario covers describe-skill behavior. + +- [ ] **Step 1: Confirm by reading the file** + +```bash +cat /Users/jesse/Documents/GitHub/superpowers/superpowers/tests/claude-code/test-subagent-driven-development.sh +``` + +Expected: tests asking the agent to describe SDD skills, not exercise them. + +- [ ] **Step 2: KEEP and annotate** + +Add at the top: + +```bash +# No drill coverage: this test asks the agent to *describe* SDD +# (asserts that asked-about skills can be summarized correctly). +# Drill scenarios test behavior, not description. Kept. +``` + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git add tests/claude-code/test-subagent-driven-development.sh +git commit -m "tests: annotate SDD describe-skill test with kept-by-design note + +Tests agent's ability to *describe* the SDD skill — drill scenarios +test behavior, not description. No drill coverage; kept by design." +``` + +--- + +## Task 11: Stale-reference scrub + +**Files:** +- Possibly modify: `docs/testing.md`, `README.md`, `CLAUDE.md`, `lefthook.yml`, `.opencode/INSTALL.md`, `.codex-plugin/INSTALL.md`, `.github/*`, `scripts/*` +- Annotate (do not rewrite): `RELEASE-NOTES.md`, `docs/superpowers/plans/*.md` + +- [ ] **Step 1: Build list of deleted-file paths** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git diff --name-only --diff-filter=D dev..HEAD | sort > /tmp/deleted-paths.txt +cat /tmp/deleted-paths.txt +``` + +- [ ] **Step 2: Search for active references** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +while read -r path; do + echo "=== $path ===" + grep -rln "$path" \ + --include="*.md" \ + --include="*.yml" \ + --include="*.yaml" \ + --include="*.sh" \ + --include="*.json" \ + --exclude-dir=node_modules \ + --exclude-dir=.venv \ + --exclude-dir=evals \ + --exclude-dir=.git \ + . +done < /tmp/deleted-paths.txt +``` + +This finds every reference to a deleted file. Categorize each hit: + +| Hit location | Treatment | +|--------------|-----------| +| `docs/testing.md` | Update — actively documents the test | +| `README.md` (Contributing section) | Update if it points at deleted tests | +| `CLAUDE.md`, `GEMINI.md`, `AGENTS.md` | Update if they reference deleted tests | +| `.github/workflows/*.yml` | Update — CI shouldn't try to run deleted tests | +| `scripts/*` | Update if they run deleted tests | +| `.opencode/INSTALL.md`, `.codex-plugin/INSTALL.md` | Update if they reference deleted tests | +| `lefthook.yml` | Update if hooks invoke deleted tests | +| `RELEASE-NOTES.md` | Annotate, don't rewrite (dated artifact) | +| `docs/superpowers/plans/*.md` | Annotate, don't rewrite (dated artifact) | + +- [ ] **Step 3: Update active references** + +For each "Update" hit, edit the file to either: +- Remove the reference if the deleted test was the only reason it was named. +- Replace with a pointer to the drill scenario (e.g., "see `evals/scenarios/triggering-test-driven-development.yaml`"). + +- [ ] **Step 4: Annotate dated artifacts** + +For each `RELEASE-NOTES.md` or `docs/superpowers/plans/*.md` hit, add an inline annotation at the *first* hit per file: + +```markdown +> Note: this section references `tests/skill-triggering/run-all.sh` and +> related bash tests that were lifted into drill scenarios on 2026-05-06 +> (see `evals/scenarios/triggering-*.yaml`). The references are +> preserved as dated artifacts of the work this doc describes. +``` + +Don't modify the actual references — they're historical. + +- [ ] **Step 5: Dispatch subagent for second-pass scrub** + +Dispatch a `general-purpose` subagent: + +``` +Working directory: /Users/jesse/Documents/GitHub/superpowers/superpowers + +These bash test paths were deleted on the current branch; some are +already addressed, but I want a second pair of eyes: + + + +Search the entire superpowers tree (excluding evals/, node_modules/, +.venv/, .git/) for any remaining references to those paths. Report +every hit with file:line and one-sentence judgment of whether it +needs an update or is fine as-is. Do not modify files; just report. +``` + +Address every reported hit before continuing. + +- [ ] **Step 6: Commit the active updates** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git add -u # picks up edits to existing files +git commit -m "docs: update references to lifted-and-deleted bash tests + +Active references in docs/testing.md, README.md, CI workflows, etc. +now point at drill scenarios. Historical references in RELEASE-NOTES.md +and docs/superpowers/plans/*.md are annotated as dated artifacts, +not rewritten." +``` + +--- + +## Task 12: Top-level docs + +**Files:** +- Modify: `docs/testing.md` — split into "Plugin tests" + "Skill behavior evals" +- Modify: `CLAUDE.md` — add evals pointer +- Modify: `README.md` — add Contributing-section pointer +- Modify: `.gitignore` — add `evals/results/`, `evals/.venv/`, `evals/.env` + +- [ ] **Step 1: Split docs/testing.md** + +The file is currently Claude-Code-centric. Split into two top-level sections. + +Open `/Users/jesse/Documents/GitHub/superpowers/superpowers/docs/testing.md` and replace the file content with this structure (preserve the existing Plugin-test details where applicable): + +```markdown +# Testing Superpowers + +Superpowers has two distinct kinds of tests, each in its own directory: + +- **`tests/`** — does the plugin's non-LLM code work? Bash + node + python integration tests for brainstorm-server JS, OpenCode plugin loading, codex-plugin sync, and analysis utilities. +- **`evals/`** — do agents behave correctly on real LLM sessions? Python harness driving real tmux sessions of Claude Code / Codex / Gemini CLI / Copilot CLI, with an LLM actor and verifier judging skill compliance. + +## Plugin tests + +Live in `tests/`. Currently: + +- `tests/brainstorm-server/` — node test suite for the brainstorm server JS code. +- `tests/opencode/` — bash tests for OpenCode plugin loading, bootstrap caching, and tool registration. +- `tests/codex-plugin-sync/` — bash sync verification. +- `tests/claude-code/test-helpers.sh`, `analyze-token-usage.py` — utilities used by remaining bash tests. +- `tests/claude-code/test-subagent-driven-development.sh` — agent-can-describe-SDD test (no drill counterpart). +- `tests/claude-code/test-subagent-driven-development-integration.sh` — extended SDD integration with token analysis (drill covers the YAGNI subset). +- `tests/explicit-skill-requests/` — Haiku-specific, multi-turn, and skill-name-prompted tests not covered by drill. + +Run plugin tests via the relevant directory's `run-*.sh` or `npm test`. + +## Skill behavior evals + +Live in `evals/`. Drill is the harness; scenarios live at `evals/scenarios/*.yaml`. See `evals/README.md` for setup. Quick start: + +```bash +cd evals +uv sync +export ANTHROPIC_API_KEY=sk-... +uv run drill run triggering-test-driven-development -b claude +``` + +Drill scenarios are slow (3-30+ minutes each) and run real LLM sessions. They are not part of CI today; the natural follow-up is a tiered model (fast subset on PR, full sweep nightly + on-demand). +``` + +- [ ] **Step 2: Update CLAUDE.md** + +Read the current CLAUDE.md, find a spot near the project structure section, and add: + +```markdown +## Eval harness + +Skill-behavior evals live at `evals/` — see `evals/README.md`. Drill (the harness) drives real tmux sessions of Claude Code / Codex / Gemini CLI / Copilot CLI and judges skill compliance with an LLM verifier. Plugin-infrastructure tests still live at `tests/`. +``` + +- [ ] **Step 3: Update README.md** + +Find the Contributing section. Add a line: + +```markdown +- Skill-behavior tests use the eval harness at `evals/`. See `evals/README.md` for setup. Plugin-infrastructure tests live at `tests/` and run via the relevant `run-*.sh` or `npm test`. +``` + +- [ ] **Step 4: Update top-level .gitignore** + +Open `/Users/jesse/Documents/GitHub/superpowers/superpowers/.gitignore` and add at the bottom: + +``` +# Eval harness — drill ships its own gitignore at evals/.gitignore; +# these are belt-and-suspenders entries for tools that don't recurse. +evals/results/ +evals/.venv/ +evals/.env +``` + +- [ ] **Step 5: Commit** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git add docs/testing.md CLAUDE.md README.md .gitignore +git commit -m "docs: introduce evals/ as the canonical skill-behavior eval harness + +- docs/testing.md split into Plugin tests + Skill behavior evals +- CLAUDE.md adds Eval harness section pointing at evals/ +- README.md Contributing section mentions evals/ alongside tests/ +- .gitignore adds evals/{results,.venv,.env} as belt-and-suspenders + (evals/.gitignore covers these locally; root-level entries help + tooling that does not recurse into nested ignore files)." +``` + +--- + +## Task 13: Re-run smoke checks (regression gate) + +**Files:** none (validation only) + +- [ ] **Step 1: Run drill's pytest** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +unset SUPERPOWERS_ROOT +uv run pytest 2>&1 | tail -5 +``` + +Expected: all tests pass. + +- [ ] **Step 2: Run cheap drill scenario** + +```bash +set -a +source /Users/jesse/Documents/GitHub/prime-radiant-inc/sprout/.env +set +a +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/evals +unset SUPERPOWERS_ROOT +uv run drill run triggering-test-driven-development -b claude 2>&1 | tail -3 +``` + +Expected: `claude: 1 passed, 0 failed, 0 errors`. If FAIL, the docs / scrub / deletion phases broke something — bisect over the recent commits. + +- [ ] **Step 3: Run remaining plugin tests that survived** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers/tests/brainstorm-server +node server.test.js 2>&1 | tail -3 +``` + +Expected: `Results: 25 passed, 0 failed`. + +--- + +## Task 14: Final adversarial review + +**Files:** none (review only; subagent dispatches) + +- [ ] **Step 1: Build the diff for reviewers** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git log --oneline dev..HEAD +git diff dev..HEAD --stat +``` + +Capture both outputs to share with reviewers. + +- [ ] **Step 2: Dispatch two parallel subagents** + +Use the `Agent` tool with two parallel calls. Same prompt to both, with adversarial framing: + +``` +Adversarial review competition: 5 points to whoever finds the most +legitimate issues. You're competing against a parallel reviewer +assigned the identical task. + +**Branch:** f/evals-lift, in /Users/jesse/Documents/GitHub/superpowers/superpowers +**Base:** dev (currently b4363df) +**Spec:** docs/superpowers/specs/2026-05-06-lift-drill-into-evals-design.md + +This branch lifts the obra/drill repo into superpowers/evals/ and +deletes redundant bash tests that drill scenarios cover. Two prior +adversarial reviews caught issues at the spec stage; this is the +post-implementation review. + +Run: git log --oneline dev..HEAD; git diff dev..HEAD --stat + +Look hard at: +1. Did the rsync-with-excludes actually exclude what it claimed? + (find evals -name '.git' -type d should return nothing) +2. Does the lift commit message point at a real commit in obra/drill? +3. Does the SUPERPOWERS_ROOT helper actually default correctly when + the env var is unset? (cd evals && unset SUPERPOWERS_ROOT && uv + run drill list — does it work?) +4. For each deleted bash test, does the corresponding drill scenario + actually verify what the bash test asserted? Spot-check by reading + the scenario YAML. +5. Are there active references in docs/, .github/, scripts/, + lefthook.yml that still point at deleted bash test paths? +6. Did the drill pytest suite get updated for the new env-var contract, + and does it pass? +7. Did the smoke scenario actually get run after path changes? +8. Is the drill repo unchanged? (cd ../drill && git status) + +Verify before claiming. If you assert "X is broken", check on disk +first. Confidently-wrong claims count negatively. + +Report format: numbered list, each with severity (critical/important/ +minor/nitpick) and one-sentence explanation with file:line. Lead with +most serious. Cap at ~600 words. +``` + +- [ ] **Step 3: Address findings** + +For each legitimate finding from either reviewer, fix in a separate commit. Re-run smoke checks (Task 13) after fixes. + +- [ ] **Step 4: Declare a winner** + +Per the cross-platform PR pattern, count legitimate findings (false positives count negatively). Acknowledge the winner in your reply summary. + +--- + +## Task 15: Push and open PR + +**Files:** none + +- [ ] **Step 1: Push the branch** + +```bash +cd /Users/jesse/Documents/GitHub/superpowers/superpowers +git push -u origin f/evals-lift +``` + +- [ ] **Step 2: Open PR against dev with full description** + +```bash +gh pr create \ + --base dev \ + --head f/evals-lift \ + --reviewer arittr \ + --title "Lift drill into superpowers as evals/ harness" \ + --body "$(cat <<'EOF' +## What problem are you trying to solve? + +Drill — the standalone Python skill-compliance benchmark at obra/drill — is already the de facto eval harness for superpowers. The PRI-1397 commit series lifted ~22 bash tests into drill scenarios, and the most recent superpowers commit (a2292c5) explicitly removed a redundant bash test with the message "replaced by drill behavioral coverage". Drill is a sibling repo today, requiring contributors to clone two checkouts and set SUPERPOWERS_ROOT manually. This PR completes the migration: drill becomes superpowers/evals/. + +## What does this PR change? + +- Lifts the obra/drill repo into superpowers as `evals/`, with explicit rsync excludes (.git, .venv, results, .env, __pycache__, *.egg-info, .private-journal). The lift commit records the source SHA. +- Adds a `_set_superpowers_root_default()` helper to drill/cli.py so SUPERPOWERS_ROOT defaults to the parent of evals/ — no manual env-var setup. +- Drops SUPERPOWERS_ROOT from required_env in codex.yaml/gemini.yaml (the helper supplies it). Claude*.yaml keep it because they interpolate ${SUPERPOWERS_ROOT} into --plugin-dir args. +- Deletes redundant bash tests under tests/skill-triggering/, tests/explicit-skill-requests/, tests/subagent-driven-dev/, and tests/claude-code/ — gated per-file by a subagent that compared each bash test's assertions to its drill scenario's verify block. Anything not 100% covered was kept. +- docs/testing.md split into Plugin tests + Skill behavior evals. +- README.md Contributing and CLAUDE.md gain pointers to evals/. + +## Is this change appropriate for the core library? + +Yes. Cross-runtime evaluation is core to superpowers, the migration to drill scenarios was already underway in this repo, and the eval harness needs to be discoverable in-tree to be findable. + +## What alternatives did you consider? + +- Vendored copy + sync script (drill repo continues independently). Rejected: divergence risk; single-source-of-truth wins. +- git subtree merge (preserves drill history in-tree). Rejected: superpowers' git history grows by 50+ commits, the merge commit is ugly, subtrees are operationally heavy. +- Keep drill as a sibling repo and just polish docs. Rejected: doesn't solve the discoverability problem. + +## Does this PR contain multiple unrelated changes? + +No — every change supports "drill is now evals/ inside superpowers". Multiple commits for atomicity (verbatim copy, env helper, YAML updates, docs) but one direction. + +## Existing PRs + +- [x] I have reviewed all open AND closed PRs for duplicates or prior art +- Related PRs: #1486 (obra/superpowers cross-platform PR — independent; no shared file changes besides README, which has no overlap) + +## Environment tested + +| Harness | Version | Model | Model ID | +|---------|---------|-------|----------| +| Claude Code | local install | Opus | claude-opus-4-7 (1M context) | + +Drill's own pytest suite passes from the new location. `triggering-test-driven-development` drill scenario passes from `evals/` after the path-default changes. (Larger drill sweep deferred to release-cadence runs per the spec's deferred-CI policy.) + +## Evaluation + +- Initial prompt: see linked spec (`docs/superpowers/specs/2026-05-06-lift-drill-into-evals-design.md`). +- Drill's own pytest suite passes. +- One drill scenario re-run from the new location end-to-end (proves the SUPERPOWERS_ROOT default works). +- Per-deleted-file subagent verification recorded in each deletion commit's message. + +## Rigor + +- [x] If this is a skills change: this is not a skills change; it's a tooling/infrastructure migration. No behavior-shaping content modified. +- [x] Adversarial pressure-tested: two parallel reviewers on the spec; final adversarial pre-PR review on the implementation; spec already corrected for findings before implementation began. +- [x] Did not modify carefully-tuned content. + +## Human review + +- [x] A human has reviewed the COMPLETE proposed diff before submission + +## Action items after merge + +1. Archive obra/drill on GitHub (mark read-only, add README pointer to obra/superpowers/evals/). +2. The spec lists CI integration, scenario co-location with skills, and Python package rename as deferred work. Open issues for any of these you want tracked. +EOF +)" +``` + +- [ ] **Step 3: Confirm PR opened** + +```bash +gh pr view --web +``` + +Expected: browser opens to the new PR. Take a screenshot or note the URL for follow-up. + +--- + +## Verification checklist (run after Task 15) + +- [ ] `git log --oneline dev..HEAD` shows the expected commits in order +- [ ] The lift commit message records the source SHA +- [ ] `find evals -name '.git' -type d` returns no output +- [ ] `cd evals && unset SUPERPOWERS_ROOT && uv run pytest` passes +- [ ] `cd evals && unset SUPERPOWERS_ROOT && uv run drill list` returns scenarios +- [ ] `cd evals && unset SUPERPOWERS_ROOT && uv run drill run triggering-test-driven-development -b claude` passes +- [ ] `tests/brainstorm-server/server.test.js` still passes (regression gate for non-LLM tests) +- [ ] `git diff dev..HEAD docs/superpowers/plans/2026-04-06-worktree-rototill.md docs/superpowers/plans/2026-03-23-codex-app-compatibility.md RELEASE-NOTES.md` shows annotations only, no path rewrites +- [ ] `cd ../drill && git log --oneline -1` shows obra/drill is unchanged from the source SHA recorded in the lift commit +- [ ] PR body lists the post-merge archival action item diff --git a/docs/superpowers/plans/2026-05-07-pi-extension-and-evals.md b/docs/superpowers/plans/2026-05-07-pi-extension-and-evals.md new file mode 100644 index 0000000000..6272acd649 --- /dev/null +++ b/docs/superpowers/plans/2026-05-07-pi-extension-and-evals.md @@ -0,0 +1,143 @@ +# Pi Extension and Evals Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add first-class Pi package support for Superpowers and add Pi as a Drill eval backend. + +**Architecture:** The Pi package is declared in the root `package.json` and loads existing `skills/` plus a small Pi extension. The extension injects the `using-superpowers` bootstrap into provider context as a user-role message on session startup and after compaction, with Pi-specific tool mapping. Drill gains a `pi` backend, Pi session-log normalization, and tests. + +**Tech Stack:** Pi TypeScript extension API, Node built-in test runner, Drill Python eval harness, pytest. + +--- + +### Task 1: Pi package manifest and extension tests + +**Files:** +- Modify: `package.json` +- Create: `tests/pi/test-pi-extension.mjs` + +- [ ] **Step 1: Write failing package/extension tests** + +Create `tests/pi/test-pi-extension.mjs` with tests that import `extensions/superpowers.ts`, register fake Pi handlers, and assert: +- root `package.json` has `keywords` containing `pi-package` +- root `package.json` has `pi.skills: ["./skills"]` +- root `package.json` has `pi.extensions: ["./extensions/superpowers.ts"]` +- the extension registers `resources_discover`, `session_start`, `session_compact`, `context`, and `agent_end` +- startup `context` injects exactly one user-role bootstrap message +- `agent_end` clears startup injection +- `session_compact` re-enables injection +- the extension does not register `session_before_compact` + +- [ ] **Step 2: Run tests and verify RED** + +Run: `node --experimental-strip-types --test tests/pi/test-pi-extension.mjs` + +Expected: FAIL because `extensions/superpowers.ts` does not exist and `package.json` lacks the `pi` manifest. + +- [ ] **Step 3: Implement manifest fields** + +Update `package.json` with `description`, `keywords`, `pi.extensions`, and `pi.skills` while preserving existing `name`, `version`, `type`, and `main`. + +- [ ] **Step 4: Implement `extensions/superpowers.ts`** + +Create a zero-runtime-dependency extension that: +- locates the package root from `import.meta.url` +- reads `skills/using-superpowers/SKILL.md` +- strips YAML frontmatter +- appends Pi-specific tool mapping +- exposes `resources_discover` with the skills path +- marks bootstrap pending on `session_start` and `session_compact` +- injects a user-role bootstrap message in `context` +- inserts post-compact bootstrap after leading `compactionSummary` messages +- clears pending bootstrap on `agent_end` + +- [ ] **Step 5: Run tests and verify GREEN** + +Run: `node --experimental-strip-types --test tests/pi/test-pi-extension.mjs` + +Expected: PASS. + +### Task 2: Pi tool mapping reference + +**Files:** +- Create: `skills/using-superpowers/references/pi-tools.md` +- Modify: `tests/pi/test-pi-extension.mjs` + +- [ ] **Step 1: Write failing test for Pi reference doc** + +Add assertions that `skills/using-superpowers/references/pi-tools.md` exists and documents mappings for `Skill`, `Task`, `TodoWrite`, and built-in tool names. + +- [ ] **Step 2: Run tests and verify RED** + +Run: `node --experimental-strip-types --test tests/pi/test-pi-extension.mjs` + +Expected: FAIL because `pi-tools.md` does not exist. + +- [ ] **Step 3: Add Pi reference doc** + +Create `skills/using-superpowers/references/pi-tools.md` explaining Pi-native skills, optional `pi-subagents`, no canonical todo/tasklist plugin, and built-in lowercase tools. + +- [ ] **Step 4: Run tests and verify GREEN** + +Run: `node --experimental-strip-types --test tests/pi/test-pi-extension.mjs` + +Expected: PASS. + +### Task 3: Drill Pi backend and session log normalization + +**Files:** +- Create: `evals/backends/pi.yaml` +- Modify: `evals/drill/backend.py` +- Modify: `evals/drill/engine.py` +- Modify: `evals/drill/normalizer.py` +- Modify: `evals/tests/test_backend.py` +- Modify: `evals/tests/test_normalizer.py` + +- [ ] **Step 1: Write failing backend/normalizer tests** + +Add pytest coverage for: +- `load_backend("pi")` returns `family == "pi"` +- Pi backend command starts with `pi` and includes `-e ${SUPERPOWERS_ROOT}` +- `_resolve_log_dir()` for Pi points under `~/.pi/agent/sessions` +- `filter_pi_logs_by_cwd()` keeps only session files whose header `cwd` matches the scenario workdir +- `normalize_pi_logs()` extracts `toolCall` blocks from Pi assistant session entries and maps built-in lowercase tools to canonical names + +- [ ] **Step 2: Run tests and verify RED** + +Run: `uv run pytest evals/tests/test_backend.py evals/tests/test_normalizer.py -q` + +Expected: FAIL because the Pi backend and normalizer do not exist. + +- [ ] **Step 3: Add `evals/backends/pi.yaml`** + +Configure the backend to run `pi -e ${SUPERPOWERS_ROOT}`, use permissive TUI readiness, `/quit` shutdown, and Pi session log location. + +- [ ] **Step 4: Implement Pi family support** + +Update `Backend.family`, `Engine._resolve_log_dir`, `Engine._collect_tool_calls`, and `normalizer.py` with Pi log filtering and normalizing. + +- [ ] **Step 5: Run tests and verify GREEN** + +Run: `uv run pytest evals/tests/test_backend.py evals/tests/test_normalizer.py -q` + +Expected: PASS. + +### Task 4: Documentation and full verification + +**Files:** +- Modify: `README.md` +- Modify: `evals/README.md` + +- [ ] **Step 1: Document Pi install and eval backend** + +Add Pi to README quickstart/install list and add backend entry/usage to `evals/README.md`. + +- [ ] **Step 2: Run verification** + +Run: +```bash +node --experimental-strip-types --test tests/pi/test-pi-extension.mjs +uv run pytest evals/tests/test_backend.py evals/tests/test_setup.py evals/tests/test_normalizer.py -q +``` + +Expected: all tests pass. diff --git a/docs/superpowers/plans/2026-05-08-visual-companion-alpine.md b/docs/superpowers/plans/2026-05-08-visual-companion-alpine.md new file mode 100644 index 0000000000..9785f704b7 --- /dev/null +++ b/docs/superpowers/plans/2026-05-08-visual-companion-alpine.md @@ -0,0 +1,989 @@ +# Visual Companion Alpine Support Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add Alpine-backed interactivity to the existing visual companion screen path without adding a second artifact/prototype system. + +**Architecture:** Vendor one pinned Alpine 3.x browser artifact in the brainstorming skill runtime, serve it from a narrow localhost route, and load it from the existing frame template for fragment screens only. Keep the current helper/event model intact, update authoring guidance so agents use Alpine sparingly, and require evidence that the new guidance changes behavior. + +**Tech Stack:** Node.js HTTP server, plain HTML/CSS/JavaScript, vendored Alpine.js 3.15.12, shell sync tests, Superpowers skill docs. + +--- + +## Source Material + +- Spec: `docs/superpowers/specs/2026-05-08-visual-companion-alpine-design.md` +- Linear: `SUP-215` +- Current branch: `codex/explore-interactive-prototypes` +- Verified Alpine package metadata on 2026-05-08: + - Version: `3.15.12` + - Tarball: `https://registry.npmjs.org/alpinejs/-/alpinejs-3.15.12.tgz` + - npm integrity: `sha512-nJvPAQVNPdZZ0NrExJ/kzQco3ijR8LwvCOadQecllESiqT4NyZ/57sN9V2XyvhlBGAbmlKYgeWZvYdKq99ij/Q==` + - Vendored file inside tarball: `package/dist/cdn.min.js` + - SHA256 of `package/dist/cdn.min.js`: `57b37d7cae9a27d965fdae4adcc844245dfdc407e655aee85dcfff3a08036a3f` + - License: MIT + - Approval artifact: `SUP-215` + +## File Structure + +- Create: `skills/brainstorming/scripts/vendor/alpine.js` + - Exact copy of Alpine `package/dist/cdn.min.js` from the pinned npm tarball. +- Create: `skills/brainstorming/scripts/vendor/alpine.provenance.json` + - Machine-readable source URL, package version, vendored path, SHA256, approval artifact, and vendoring date. +- Create: `skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md` + - Human-readable Alpine license notice and refresh command. +- Modify: `skills/brainstorming/scripts/server.cjs` + - Add parsed-path vendor serving for `/vendor/alpine.js`. +- Modify: `skills/brainstorming/scripts/frame-template.html` + - Load Alpine for frame-wrapped fragments and neutralize the footer copy. +- Modify: `tests/brainstorm-server/server.test.js` + - Cover provenance, vendor route behavior, helper injection, frame injection, and full-document/waiting-page boundaries. +- Modify: `skills/brainstorming/visual-companion.md` + - Update agent-facing guidance from selection-first/static mockups to compact Alpine-backed interactive mockups. +- Modify: `scripts/sync-to-codex-plugin.sh` + - Surface vendored Alpine provenance in generated Codex plugin sync PR bodies. +- Modify: `tests/codex-plugin-sync/test-sync-to-codex-plugin.sh` + - Ensure nested skill-local scripts and vendor files survive root `/scripts/` exclusion and generated PR-body source includes the vendored dependency note. + +## Task 1: Vendor Alpine and Add Provenance Tests + +**Files:** +- Create: `skills/brainstorming/scripts/vendor/alpine.js` +- Create: `skills/brainstorming/scripts/vendor/alpine.provenance.json` +- Create: `skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md` +- Modify: `tests/brainstorm-server/server.test.js` + +- [ ] **Step 1: Write the failing provenance test** + +Add this import alongside the existing `require` block: + +```js +const crypto = require('crypto'); +``` + +Add these constants near the existing `SERVER_PATH`, `TEST_PORT`, and directory constants in `tests/brainstorm-server/server.test.js`: + +```js +const ALPINE_PATH = path.join(__dirname, '../../skills/brainstorming/scripts/vendor/alpine.js'); +const ALPINE_PROVENANCE_PATH = path.join(__dirname, '../../skills/brainstorming/scripts/vendor/alpine.provenance.json'); +const ALPINE_NOTICES_PATH = path.join(__dirname, '../../skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md'); +``` + +Add this helper below `fetch(url)`: + +```js +function sha256File(filePath) { + return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex'); +} +``` + +Add this test block at the start of `runTests()`, before `// ========== Server Startup ==========`: + +```js + // ========== Vendored Alpine ========== + console.log('\n--- Vendored Alpine ---'); + + await test('vendored Alpine provenance is complete and matches artifact hash', () => { + assert(fs.existsSync(ALPINE_PATH), 'alpine.js should exist'); + assert(fs.existsSync(ALPINE_PROVENANCE_PATH), 'alpine.provenance.json should exist'); + assert(fs.existsSync(ALPINE_NOTICES_PATH), 'THIRD_PARTY_NOTICES.md should exist'); + + const provenance = JSON.parse(fs.readFileSync(ALPINE_PROVENANCE_PATH, 'utf-8')); + assert.strictEqual(provenance.name, 'alpinejs'); + assert.strictEqual(provenance.version, '3.15.12'); + assert.strictEqual(provenance.license, 'MIT'); + assert.strictEqual(provenance.sourceUrl, 'https://registry.npmjs.org/alpinejs/-/alpinejs-3.15.12.tgz'); + assert.strictEqual(provenance.sourcePackagePath, 'package/dist/cdn.min.js'); + assert.strictEqual(provenance.localPath, 'skills/brainstorming/scripts/vendor/alpine.js'); + assert.strictEqual(provenance.sha256, '57b37d7cae9a27d965fdae4adcc844245dfdc407e655aee85dcfff3a08036a3f'); + assert.strictEqual(provenance.approvalArtifact, 'SUP-215'); + assert.strictEqual(sha256File(ALPINE_PATH), provenance.sha256); + + const notices = fs.readFileSync(ALPINE_NOTICES_PATH, 'utf-8'); + assert(notices.includes('Alpine.js'), 'Notice should name Alpine.js'); + assert(notices.includes('MIT License'), 'Notice should include MIT license text'); + assert(notices.includes('curl -fsSL https://registry.npmjs.org/alpinejs/-/alpinejs-3.15.12.tgz'), 'Notice should include refresh command'); + return Promise.resolve(); + }); +``` + +- [ ] **Step 2: Run the failing test** + +Run: + +```bash +cd "$(git rev-parse --show-toplevel)" +node tests/brainstorm-server/server.test.js +``` + +Expected: FAIL with `alpine.js should exist`. + +- [ ] **Step 3: Vendor Alpine from the pinned npm tarball** + +Run: + +```bash +cd /Users/drewritter/prime-rad/superpowers +mkdir -p skills/brainstorming/scripts/vendor +tmpdir="$(mktemp -d)" +curl -fsSL https://registry.npmjs.org/alpinejs/-/alpinejs-3.15.12.tgz -o "$tmpdir/alpinejs-3.15.12.tgz" +tar -xzf "$tmpdir/alpinejs-3.15.12.tgz" -C "$tmpdir" package/dist/cdn.min.js +cp "$tmpdir/package/dist/cdn.min.js" skills/brainstorming/scripts/vendor/alpine.js +rm -rf "$tmpdir" +shasum -a 256 skills/brainstorming/scripts/vendor/alpine.js +``` + +Expected SHA256: + +```text +57b37d7cae9a27d965fdae4adcc844245dfdc407e655aee85dcfff3a08036a3f +``` + +- [ ] **Step 4: Create provenance metadata** + +Create `skills/brainstorming/scripts/vendor/alpine.provenance.json` with this exact JSON: + +```json +{ + "name": "alpinejs", + "version": "3.15.12", + "license": "MIT", + "sourceUrl": "https://registry.npmjs.org/alpinejs/-/alpinejs-3.15.12.tgz", + "sourcePackagePath": "package/dist/cdn.min.js", + "localPath": "skills/brainstorming/scripts/vendor/alpine.js", + "npmIntegrity": "sha512-nJvPAQVNPdZZ0NrExJ/kzQco3ijR8LwvCOadQecllESiqT4NyZ/57sN9V2XyvhlBGAbmlKYgeWZvYdKq99ij/Q==", + "sha256": "57b37d7cae9a27d965fdae4adcc844245dfdc407e655aee85dcfff3a08036a3f", + "approvalArtifact": "SUP-215", + "vendoredAt": "2026-05-08" +} +``` + +Create `skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md` with: + +````markdown +# Third-Party Notices + +## Alpine.js + +- Package: `alpinejs` +- Version: `3.15.12` +- Source: `https://registry.npmjs.org/alpinejs/-/alpinejs-3.15.12.tgz` +- Vendored file: `package/dist/cdn.min.js` +- Local path: `skills/brainstorming/scripts/vendor/alpine.js` +- SHA256: `57b37d7cae9a27d965fdae4adcc844245dfdc407e655aee85dcfff3a08036a3f` + +Refresh command: + +```bash +cd "$(git rev-parse --show-toplevel)" +tmpdir="$(mktemp -d)" +curl -fsSL https://registry.npmjs.org/alpinejs/-/alpinejs-3.15.12.tgz -o "$tmpdir/alpinejs-3.15.12.tgz" +tar -xzf "$tmpdir/alpinejs-3.15.12.tgz" -C "$tmpdir" package/dist/cdn.min.js +cp "$tmpdir/package/dist/cdn.min.js" skills/brainstorming/scripts/vendor/alpine.js +shasum -a 256 skills/brainstorming/scripts/vendor/alpine.js +rm -rf "$tmpdir" +``` + +License: + +```text +MIT License + +Copyright © 2019-2025 Caleb Porzio and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` +```` + +- [ ] **Step 5: Run the provenance test** + +Run: + +```bash +cd /Users/drewritter/prime-rad/superpowers +node tests/brainstorm-server/server.test.js +``` + +Expected: the vendored Alpine provenance test passes. Later HTTP tests may still fail until Task 2 if they have already been added; do not commit until this command exits 0 after Task 2. + +- [ ] **Step 6: Commit Task 1** + +After Task 2 also passes the full server test, commit Task 1 and Task 2 together. The vendored file and server route are one behavioral unit. + +## Task 2: Serve Alpine and Inject It Into Frame-Wrapped Fragments + +**Files:** +- Modify: `skills/brainstorming/scripts/server.cjs` +- Modify: `skills/brainstorming/scripts/frame-template.html` +- Modify: `tests/brainstorm-server/server.test.js` + +- [ ] **Step 1: Add failing HTTP and injection tests** + +Add this test after `returns Content-Type text/html`: + +```js + await test('waiting page does not inject Alpine', async () => { + const res = await fetch(`http://localhost:${TEST_PORT}/`); + assert(!res.body.includes('/vendor/alpine.js'), 'Waiting page should not inject Alpine'); + }); +``` + +Add these tests after `returns 404 for non-root paths`: + +```js + await test('serves vendored Alpine from exact vendor route', async () => { + const res = await fetch(`http://localhost:${TEST_PORT}/vendor/alpine.js`); + assert.strictEqual(res.status, 200); + assert(res.headers['content-type'].includes('application/javascript'), 'Should be JavaScript'); + assert(res.body.includes('Alpine'), 'Should serve Alpine script content'); + }); + + await test('serves vendored Alpine when query string is present', async () => { + const res = await fetch(`http://localhost:${TEST_PORT}/vendor/alpine.js?v=3.15.12`); + assert.strictEqual(res.status, 200); + assert(res.body.includes('Alpine'), 'Should ignore query string for exact vendor pathname'); + }); + + await test('exact-match vendor route rejects non-allowlisted pathnames', async () => { + const paths = [ + '/vendor/unknown.js', + '/vendor/alpine.js/extra', + '/vendor/../alpine.js', + '/vendor/%2e%2e/alpine.js', + '/vendor/%2E%2E/alpine.js' + ]; + + for (const requestPath of paths) { + const res = await fetch(`http://localhost:${TEST_PORT}${requestPath}`); + assert.strictEqual(res.status, 404, `${requestPath} should 404`); + } + }); +``` + +This test should assert the actual defense: the route is an exact parsed-pathname +allowlist. Do not describe `/vendor/../alpine.js` as proving filesystem +canonicalization, because the URL parser normalizes that request before the +vendor allowlist sees it. + +Update `serves full HTML documents as-is (not wrapped)` with this assertion: + +```js + assert(!res.body.includes('/vendor/alpine.js'), 'Should NOT inject Alpine into full documents'); +``` + +Update `wraps content fragments in frame template` with these assertions: + +```js + assert(res.body.includes(''), 'Fragment should load Alpine'); + assert(res.body.includes('Interact with the mockup, then return to the terminal'), 'Frame copy should be neutral'); +``` + +Add this test after `wraps content fragments in frame template`: + +```js + await test('preserves Alpine attributes in frame-wrapped fragments', async () => { + const fragment = '
Details
'; + fs.writeFileSync(path.join(CONTENT_DIR, 'alpine-fragment.html'), fragment); + await sleep(300); + + const res = await fetch(`http://localhost:${TEST_PORT}/`); + assert(res.body.includes('x-data="{ open: false }"'), 'Should preserve x-data'); + assert(res.body.includes('@click="open = !open"'), 'Should preserve @click'); + assert(res.body.includes('x-show="open"'), 'Should preserve x-show'); + assert(res.body.includes('/vendor/alpine.js'), 'Should include Alpine script'); + }); +``` + +- [ ] **Step 2: Run the failing tests** + +Run: + +```bash +cd /Users/drewritter/prime-rad/superpowers +node tests/brainstorm-server/server.test.js +``` + +Expected: FAIL because `/vendor/alpine.js` returns 404 and the frame does not include Alpine yet. + +- [ ] **Step 3: Implement exact vendor serving** + +In `skills/brainstorming/scripts/server.cjs`, add these constants after `helperInjection`: + +```js +const ALPINE_VENDOR_PATH = path.join(__dirname, 'vendor', 'alpine.js'); + +function loadVendorFile(filePath, name) { + try { + return fs.readFileSync(filePath); + } catch (error) { + throw new Error( + `Failed to load vendored ${name} at ${filePath}; ` + + 'run the refresh command in skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md. ' + + error.message + ); + } +} + +const VENDOR_FILES = new Map([ + ['/vendor/alpine.js', { + content: loadVendorFile(ALPINE_VENDOR_PATH, 'Alpine'), + contentType: 'application/javascript; charset=utf-8' + }] +]); +``` + +Add these helpers after `getNewestScreen()`: + +```js +function parseRequestUrl(req) { + return new URL(req.url, 'http://localhost'); +} + +function serveVendorFile(requestUrl, res) { + const vendorFile = VENDOR_FILES.get(requestUrl.pathname); + if (!vendorFile) { + res.writeHead(404); + res.end('Not found'); + return; + } + + res.writeHead(200, { 'Content-Type': vendorFile.contentType }); + res.end(vendorFile.content); +} +``` + +Change the start of `handleRequest(req, res)` to parse once and use `pathname`: + +```js +function handleRequest(req, res) { + touchActivity(); + const requestUrl = parseRequestUrl(req); + + if (req.method === 'GET' && requestUrl.pathname === '/') { +``` + +Add the vendor branch before `/files/`: + +```js + } else if (req.method === 'GET' && requestUrl.pathname.startsWith('/vendor/')) { + serveVendorFile(requestUrl, res); + } else if (req.method === 'GET' && requestUrl.pathname.startsWith('/files/')) { + const fileName = requestUrl.pathname.slice(7); +``` + +Keep the rest of the `/files/` branch unchanged except that it now uses `fileName` from `requestUrl.pathname`. + +- [ ] **Step 4: Inject Alpine from the frame template** + +In `skills/brainstorming/scripts/frame-template.html`, add this script tag immediately before ``: + +```html + +``` + +Change the indicator copy to: + +```html + Interact with the mockup, then return to the terminal +``` + +- [ ] **Step 5: Run the server tests** + +Run: + +```bash +cd /Users/drewritter/prime-rad/superpowers +node tests/brainstorm-server/server.test.js +``` + +Expected: `PASS` and `0 failed`. + +- [ ] **Step 6: Commit Tasks 1 and 2** + +Run: + +```bash +cd /Users/drewritter/prime-rad/superpowers +git add \ + skills/brainstorming/scripts/server.cjs \ + skills/brainstorming/scripts/frame-template.html \ + skills/brainstorming/scripts/vendor/alpine.js \ + skills/brainstorming/scripts/vendor/alpine.provenance.json \ + skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md \ + tests/brainstorm-server/server.test.js +git commit -m "feat: add Alpine to visual companion runtime" +``` + +## Task 3: Preserve Alpine Through Codex Plugin Sync + +**Files:** +- Modify: `scripts/sync-to-codex-plugin.sh` +- Modify: `tests/codex-plugin-sync/test-sync-to-codex-plugin.sh` + +- [ ] **Step 1: Add failing sync fixture coverage** + +In `write_upstream_fixture()`, extend the `mkdir -p` block with: + +```bash + "$repo/skills/brainstorming/scripts/vendor" \ +``` + +After the example skill fixture, add: + +```bash + cat > "$repo/skills/brainstorming/scripts/server.cjs" <<'EOF' +console.log('fixture server') +EOF + + cat > "$repo/skills/brainstorming/scripts/helper.js" <<'EOF' +window.fixtureHelper = true +EOF + + cat > "$repo/skills/brainstorming/scripts/frame-template.html" <<'EOF' + +EOF + + printf 'fixture alpine\n' > "$repo/skills/brainstorming/scripts/vendor/alpine.js" + + cat > "$repo/skills/brainstorming/scripts/vendor/alpine.provenance.json" <<'EOF' +{"name":"alpinejs","version":"3.15.12","localPath":"skills/brainstorming/scripts/vendor/alpine.js","sha256":"fixture","approvalArtifact":"SUP-215"} +EOF + + cat > "$repo/skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md" <<'EOF' +# Third-Party Notices + +Alpine.js fixture notice. +EOF +``` + +Add these paths to the `git -C "$repo" add` list: + +```bash + skills/brainstorming/scripts/server.cjs \ + skills/brainstorming/scripts/helper.js \ + skills/brainstorming/scripts/frame-template.html \ + skills/brainstorming/scripts/vendor/alpine.js \ + skills/brainstorming/scripts/vendor/alpine.provenance.json \ + skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md \ +``` + +In `write_synced_destination_fixture()`, extend the `mkdir -p` block with: + +```bash + "$repo/plugins/superpowers/skills/brainstorming/scripts/vendor" \ +``` + +Add the same fixture files under `plugins/superpowers/skills/brainstorming/scripts/`, then add those paths to the destination `git add` list. + +Add these preview assertions after `Preview reflects dirty tracked destination file`: + +```bash + assert_contains "$preview_section" "skills/brainstorming/scripts/server.cjs" "Preview includes skill-local server runtime" + assert_contains "$preview_section" "skills/brainstorming/scripts/helper.js" "Preview includes skill-local helper runtime" + assert_contains "$preview_section" "skills/brainstorming/scripts/frame-template.html" "Preview includes skill-local frame template" + assert_contains "$preview_section" "skills/brainstorming/scripts/vendor/alpine.js" "Preview includes vendored Alpine" + assert_contains "$preview_section" "skills/brainstorming/scripts/vendor/alpine.provenance.json" "Preview includes Alpine provenance" + assert_contains "$preview_section" "skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md" "Preview includes Alpine notice" +``` + +Add these no-op fixture path variables near `noop_openai_metadata_path`: + +```bash + local noop_alpine_path + local noop_alpine_provenance_path + local noop_alpine_notice_path +``` + +Assign them after `noop_openai_metadata_path=...`: + +```bash + noop_alpine_path="$noop_apply_dest/plugins/superpowers/skills/brainstorming/scripts/vendor/alpine.js" + noop_alpine_provenance_path="$noop_apply_dest/plugins/superpowers/skills/brainstorming/scripts/vendor/alpine.provenance.json" + noop_alpine_notice_path="$noop_apply_dest/plugins/superpowers/skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md" +``` + +Add these no-op assertions after the OpenAI metadata assertion: + +```bash + assert_file_equals "$noop_alpine_path" "fixture alpine" "Clean no-op local apply preserves vendored Alpine" + assert_file_equals "$noop_alpine_provenance_path" "{\"name\":\"alpinejs\",\"version\":\"3.15.12\",\"localPath\":\"skills/brainstorming/scripts/vendor/alpine.js\",\"sha256\":\"fixture\",\"approvalArtifact\":\"SUP-215\"}" "Clean no-op local apply preserves Alpine provenance" + assert_contains "$(cat "$noop_alpine_notice_path")" "Alpine.js fixture notice." "Clean no-op local apply preserves Alpine notice" +``` + +Add this source assertion near the existing source assertions: + +```bash + assert_contains "$script_source" "Vendored third-party code included in this sync" "Source calls out vendored third-party code in sync PR body" +``` + +- [ ] **Step 2: Run the failing sync test** + +Run: + +```bash +cd /Users/drewritter/prime-rad/superpowers +bash tests/codex-plugin-sync/test-sync-to-codex-plugin.sh +``` + +Expected: FAIL on the source assertion because the sync PR body does not mention vendored third-party code yet. + +- [ ] **Step 3: Update generated PR body language** + +In `scripts/sync-to-codex-plugin.sh`, add this helper before +`if [[ $BOOTSTRAP -eq 1 ]]; then` in the commit/PR section. Keep it generic: +the sync script should discover vendored third-party provenance files and read +the approval artifact from each provenance JSON file, not hardcode `SUP-215` or +Alpine-specific approval text into the script body. + +```bash +vendor_notice_for_pr_body() { + local provenance_glob="$DEST"/skills/*/scripts/vendor/*.provenance.json + + if ! compgen -G "$provenance_glob" > /dev/null; then + return 0 + fi + + python3 - "$DEST" <<'PY' +import glob +import json +import os +import sys + +dest = sys.argv[1] +provenance_files = sorted(glob.glob(os.path.join(dest, "skills", "*", "scripts", "vendor", "*.provenance.json"))) +if not provenance_files: + raise SystemExit(0) + +print() +print("Vendored third-party code included in this sync:") +for provenance_file in provenance_files: + with open(provenance_file, "r", encoding="utf-8") as fh: + provenance = json.load(fh) + + rel_provenance = os.path.relpath(provenance_file, dest) + rel_vendor_dir = os.path.dirname(rel_provenance) + basename = os.path.basename(provenance_file).removesuffix(".provenance.json") + local_path = provenance.get("localPath") or os.path.join(rel_vendor_dir, f"{basename}.js") + notice_path = os.path.join(rel_vendor_dir, "THIRD_PARTY_NOTICES.md") + name = provenance.get("name", "unknown") + version = provenance.get("version", "unknown") + approval = provenance.get("approvalArtifact", "not recorded") + sha256 = provenance.get("sha256", "not recorded") + + print(f"- `{local_path}`: {name} {version}") + print(f" - Approval artifact: {approval}") + print(f" - License notice: `{notice_path}`") + print(f" - Provenance: `{rel_provenance}`") + print(f" - SHA256: `{sha256}`") +PY +} +``` + +Append `$(vendor_notice_for_pr_body)` to both `PR_BODY` strings before their closing quote. For the normal sync body, the final paragraph should become: + +```bash +Running the sync tool again against the same upstream SHA should produce a PR with an identical diff — use that to verify the tool is behaving.$(vendor_notice_for_pr_body)" +``` + +For the bootstrap body, the final paragraph should become: + +```bash +This is a one-time bootstrap. Subsequent syncs will be normal (non-bootstrap) runs using the same tracked upstream plugin files.$(vendor_notice_for_pr_body)" +``` + +- [ ] **Step 4: Run the sync test** + +Run: + +```bash +cd /Users/drewritter/prime-rad/superpowers +bash tests/codex-plugin-sync/test-sync-to-codex-plugin.sh +``` + +Expected: `PASS`. + +- [ ] **Step 5: Commit Task 3** + +Run: + +```bash +cd /Users/drewritter/prime-rad/superpowers +git add scripts/sync-to-codex-plugin.sh tests/codex-plugin-sync/test-sync-to-codex-plugin.sh +git commit -m "test: cover Alpine in Codex plugin sync" +``` + +## Task 4: Update Visual Companion Guidance + +**Files:** +- Modify: `skills/brainstorming/visual-companion.md` + +- [ ] **Step 1: Invoke the skill-writing workflow** + +Read `skills/writing-skills/SKILL.md` before editing `visual-companion.md`. + +- [ ] **Step 2: Update the selection-first copy** + +Change the `How It Works` paragraph to: + +```markdown +The server watches a directory for HTML files and serves the newest one to the browser. You write HTML content to `screen_dir`, the user tries the mockup in their browser, and they respond in the terminal. Use `[data-choice]` only when you are deliberately asking the user to pick among named A/B/C visual options. +``` + +Change Loop step 2 to: + +```markdown +2. **Tell user what to expect and end your turn:** + - Remind them of the URL (every step, not just first) + - Give a brief text summary of what's on screen (e.g., "Showing an interactive meal-planning mockup with tabs and an editable grocery list") + - Ask them to respond in the terminal: "Take a look, try the mockup, and tell me what feels right or wrong." + - If the screen is a deliberate A/B/C choice, also say: "Click an option if you'd like; your terminal feedback is still the source of truth." +``` + +- [ ] **Step 3: Add compact Alpine guidance before the current minimal example** + +Insert this section before `**Minimal example:**`: + +````markdown +## Interactive Mockups With Alpine + +Frame-wrapped fragments automatically load Alpine.js. Use Alpine when visible interaction is central to the design question: tabs, toggles, accordions, modal open/close, wizard next/back, lightweight form validation, or simple add/remove list behavior. + +Keep it illustrative. Do not build a fake application just because realistic chrome includes many controls. If an interaction is not part of the question, render that area as passive content. + +```html +
+
+ + +
+ +
+

Week plan

+

Three realistic meals are enough for the mockup.

+
+ +
+

Grocery list

+
    + +
+ + +
+
+``` + +Rules: + +- Write content fragments by default; do not add an Alpine `` to + `frame-template.html`. +- Keep the existing helper server-injected from `server.cjs` into every served + page, including waiting pages and full HTML documents. +- Do not automatically inject Alpine into waiting pages or full HTML documents. + Full documents may include their own scripts, including `/vendor/alpine.js`, + when they need complete control. +- Update the frame's default indicator copy from a selection-specific prompt to + neutral language such as "Interact with the mockup, then return to the + terminal." Preserve the helper's selected-choice update behavior when a + deliberate `[data-choice]` is clicked. + +Required runtime invariant: + +- By the time `DOMContentLoaded` fires for a served frame-wrapped fragment, + every `x-data` block in that fragment has been evaluated and `x-show` / + `@click` directives are bound. +- The existing helper must still connect to the WebSocket server, reload on + screen changes, and capture deliberate `[data-choice]` clicks. +- The helper must not depend on Alpine. + +Expected served fragment order: + +1. Page/frame HTML +2. Alpine script with `defer` +3. Existing helper injection + +Because `defer` changes execution order, the implementation should test the +runtime behavior rather than only checking byte order in the served HTML. + +V1 guarantees automatic Alpine support only for normal frame-wrapped fragments. +The common agent path should remain fragments; do not require robust +full-document Alpine injection in SUP-215. + +### Codex Plugin Sync + +The root sync script already uses anchored root-level excludes, so `/scripts/` +does not match nested skill-local paths like +`skills/brainstorming/scripts/vendor/alpine.js`. SUP-215 should preserve that +behavior rather than changing the exclusion model. + +The sync script does need one user-visible change: generated Codex plugin PR +bodies should surface the vendored third-party code when the synced diff +includes `skills/brainstorming/scripts/vendor/alpine.js`. The PR body should +call out the approval artifact, license notice, and SHA256 provenance instead +of presenting the sync as an opaque tracked-file copy. + +### Mockup Authoring Guidance + +Update `visual-companion.md` so agents treat Alpine as available by default. + +The key instruction: + +> If a visual mockup includes something that looks clickable, editable, or +> selectable to a user, make it work only when that interaction is part of the +> current design question. Otherwise, render it visibly as passive non-control +> content or keep the behavior minimal and illustrative. + +The guide should lead with an Alpine-backed interactive mockup example before +the existing selection-card examples. Existing `data-choice` examples should be +kept but clearly labeled as deliberate A/B choice affordances, not normal UI +controls. + +Keep the guide compact. It should include one concise Alpine example and a +terse do/don't checklist, not a cookbook of separate snippets for every UI +pattern. + +Common Alpine patterns the example or checklist may reference: + +- tabs and sidebar navigation +- modal/dialog open and close +- accordion expand/collapse +- form input and lightweight validation +- multi-step wizard navigation +- toggle/switch state +- simple list add/remove/edit behavior +- toast or inline success feedback + +Controls that should work when they are central to the current visual question: + +- tabs and sidebar/nav items +- buttons that imply state changes +- toggles and switches +- form fields and submit buttons +- modal/dialog triggers +- accordion headers +- wizard next/back controls +- add/edit/delete list actions + +Boundaries: + +These are authoring rules enforced by agent discipline, skill guidance, human +review, and eval evidence. They are not enforced by the server, frame template, +or vendored Alpine in V1. If runtime enforcement becomes necessary, that should +be a follow-up hardening task, likely involving CSP and a revisit of the Alpine +CSP build. + +- No fake backend calls. +- No network requests. +- No localStorage/sessionStorage persistence. +- No complex application logic beyond what the mockup needs to communicate. +- No interactivity that is not visually implied by the mockup. +- Do not build full add/edit/delete/search/wizard behavior merely because those + controls appear in a realistic product screen. If the question is about visual + hierarchy, surrounding app chrome can be passive. +- No script tags for Alpine; the frame provides it. +- Do not put exploratory Alpine controls inside `[data-choice]` containers + unless the click is intended to select that choice. Use a separate choice + affordance or `@click.stop` where appropriate. +- Replace existing network-positive guidance such as loading live Unsplash + images. If real images matter, use project-provided local assets through the + existing `/files/` route or choose a simple local placeholder. + +### Sample Data Policy + +Do not ship canned sample fixtures. + +When a mockup represents data, the agent should create 2-5 compact, realistic, +domain-specific records. The records should match the product being discussed. +A family meal-planning tool should not show generic SaaS users; a workshop +scheduling app should show realistic sessions, facilitators, rooms, or dates. + +Put records in Alpine `x-data` only when interaction needs state, such as +filtering, editing, adding, selecting, or stepping through records. If the data +is only presentational, render it directly as HTML. + +This keeps mockups grounded in the user's idea and avoids every screen +collapsing into the same dashboard template. + +### Feedback and Events + +V1 keeps the current feedback model unchanged. + +- The terminal remains the primary feedback channel. +- Existing `[data-choice]` click capture remains supported. +- Alpine interactions are for user understanding, not automatic telemetry. +- Default guide and frame language should say "try/interact with the mockup, + then respond in the terminal," not "click an option" unless the screen is + explicitly asking for an A/B/C choice. +- Use `data-choice` only when asking the user to choose among named options the + agent should read on the next turn. +- Do not instrument ordinary tabs, forms, toggles, modals, or list interactions + as choice events. +- Do not add broad interaction streaming in V1. +- Do not ask agents to wire new `brainstorm.feedback(...)` calls in V1. + +This avoids expanding context with noisy interaction logs. The user can freely +poke at a mockup, then tell the agent what worked or did not work. + +## V2 Follow-Up + +After dogfooding Alpine-backed mockups, revisit the old selection-oriented +event model. + +Possible V2 direction: + +- Remove or de-emphasize the selection-specific helper code. +- Replace it with a general ephemeral interaction stream file. +- Keep that stream out of default context; agents should read it only when it is + useful. +- Clear the stream when a new screen is pushed and/or when the server stops. + +Do not implement this in SUP-215. The point of V1 is to learn whether Alpine +improves visual brainstorming before changing the feedback model. + +## Security and Trust Boundary + +Superpowers visual companion is not Brainstorm. + +Brainstorm renders user-generated artifacts inside a multi-user web +application, so CSP and iframe sandboxing are product security boundaries. +Superpowers runs a local helper server inside the user's coding harness. The +server binds to `127.0.0.1` by default, and the user has already authorized the +agent to write local files and run local commands. + +The relevant V1 guardrails are: + +- keep the default bind host as localhost-only +- vendor Alpine instead of fetching it from a CDN at runtime +- serve only known vendored files +- prohibit network requests in generated mockups +- prohibit storage-based persistence in generated mockups + +CSP and iframe sandboxing can be revisited if local usage reveals a concrete +need. + +## Testing + +Extend the existing brainstorm server tests. + +Required coverage: + +- `/vendor/alpine.js` returns the vendored Alpine script with a JavaScript + content type. +- `/vendor/alpine.js?v=` returns the same vendored script. +- Unknown, nested, and traversal-ish vendor paths return 404, including encoded + traversal attempts. +- Frame-wrapped fragments include the Alpine script automatically. +- Existing helper injection still occurs. +- Waiting pages and full HTML documents continue to receive helper injection + and do not receive automatic Alpine injection. +- Existing `[data-choice]` click capture still writes `state/events`. +- A fragment containing Alpine attributes is served without stripping or + escaping those attributes. +- Vendored Alpine provenance verification recomputes the SHA256 and checks the + required metadata and notice files. + +Do not pretend the existing `tests/brainstorm-server/server.test.js` harness can +prove Alpine runtime behavior. It is an HTTP/WebSocket test harness and does not +execute browser DOM events or Alpine directives. Runtime behaviors such as +`x-show`, `@click`, and `@click.stop` must be covered by a real browser test if +one is added, or by manual dogfood evidence in the PR. + +Codex plugin sync coverage: + +- Update `tests/codex-plugin-sync/test-sync-to-codex-plugin.sh` so the fixture + includes the visual companion runtime files: + `skills/brainstorming/scripts/server.cjs`, + `skills/brainstorming/scripts/helper.js`, + `skills/brainstorming/scripts/frame-template.html`, + `skills/brainstorming/scripts/vendor/alpine.js`, + `skills/brainstorming/scripts/vendor/alpine.provenance.json`, and + `skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md`. +- Assert that dry-run preview includes those nested skill-local runtime files. +- Assert that the no-op synced destination fixture contains those files, so the + test proves root `/scripts/` exclusion does not remove + `skills/brainstorming/scripts/`. +- If a positive changed-apply fixture is added, assert that the applied + destination contains the vendored Alpine file and provenance files. +- Update `scripts/sync-to-codex-plugin.sh` PR body generation so any downstream + Codex plugin PR carrying `skills/brainstorming/scripts/vendor/alpine.js` + explicitly calls out the vendored third-party code, approval artifact, + license notice, and SHA256 provenance. + +Skill behavior coverage: + +- Use `superpowers:writing-skills` for the `visual-companion.md` behavior + change. +- Include adversarial pressure-test evidence in the implementation PR: initial + prompt, environment, eval count, observed output, and whether the output met + expectations. +- Cover at least this matrix: + - Interactive mockup without `data-choice`: uses Alpine directives, omits an + Alpine script tag, includes compact domain-specific sample data when useful, + avoids backend/storage/network behavior, and asks the user to respond in the + terminal. + - Deliberate A/B choice: preserves `data-choice` for named options and keeps + the choice semantics clear. + - Static visual: uses no Alpine when interactivity is not useful. + - Busy dashboard or app shell: limits interactivity to the design question and + does not build a fake mini-application. + - Image-heavy mockup that previously might have used a live Unsplash URL: now + uses a `/files/` local asset or a local placeholder, with + before/after evidence for the guidance change. + +Manual dogfood check: + +1. Start the visual companion with `scripts/start-server.sh --project-dir`. +2. Write a normal fragment that uses `x-data`, `@click`, and `x-show`. +3. Open the local URL. +4. Confirm Alpine initializes with no console errors. +5. Confirm `@click` changes state and `x-show` toggles visibility. +6. Confirm the interaction works without the agent adding an Alpine script tag. +7. Confirm a nested Alpine control using `@click.stop` near a `[data-choice]` + surface does not produce an unintended extra choice event. +8. Confirm the terminal remains the feedback path. + +If adding an automated browser dependency is too heavy for SUP-215, this +browser proof can be manual PR evidence rather than a new test dependency. + +## Rollout + +V1 is an experiment, but it should still ship cleanly: + +- Keep changes contained to the brainstorming skill runtime, guide, and tests. +- Do not change the visual companion startup flow. +- Do not create a new mode in the user-facing language. +- Describe the behavior as "interactive mockups" or "Alpine-backed mockups," + not as a separate artifact/prototype system. +- Include the maintainer-approved dependency exception and third-party + provenance in the PR. +- Include real browser dogfood evidence that Alpine initializes and runs. +- Include skill-behavior evidence that the updated guidance changes agent + output, not just server bytes. +- Include the PR base in the review notes. The SUP-215 PR should show a focused + diff against its chosen base. +- After dogfooding, decide whether SUP-215 should be followed by a V2 ticket + for event-stream cleanup. diff --git a/docs/testing.md b/docs/testing.md index c283e78e0a..7bde5a0869 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,303 +1,34 @@ -# Testing Superpowers Skills +# Testing Superpowers -This document describes how to test Superpowers skills, particularly the integration tests for complex skills like `subagent-driven-development`. +Superpowers has two distinct kinds of tests, each in its own directory: -## Overview +- **`tests/`** — does the plugin's non-LLM code work? Bash + node + python integration tests for brainstorm-server JS, OpenCode plugin loading, codex-plugin sync, and analysis utilities. +- **`evals/`** — do agents behave correctly on real LLM sessions? Python harness driving real tmux sessions of Claude Code / Codex / Gemini CLI, with an LLM actor and verifier judging skill compliance. -Testing skills that involve subagents, workflows, and complex interactions requires running actual Claude Code sessions in headless mode and verifying their behavior through session transcripts. +## Plugin tests -## Test Structure +Live in `tests/`. Currently: -``` -tests/ -├── claude-code/ -│ ├── test-helpers.sh # Shared test utilities -│ ├── test-subagent-driven-development-integration.sh -│ ├── analyze-token-usage.py # Token analysis tool -│ └── run-skill-tests.sh # Test runner (if exists) -``` - -## Running Tests - -### Integration Tests - -Integration tests execute real Claude Code sessions with actual skills: - -```bash -# Run the subagent-driven-development integration test -cd tests/claude-code -./test-subagent-driven-development-integration.sh -``` - -**Note:** Integration tests can take 10-30 minutes as they execute real implementation plans with multiple subagents. - -### Requirements - -- Must run from the **superpowers plugin directory** (not from temp directories) -- Claude Code must be installed and available as `claude` command -- Local dev marketplace must be enabled: `"superpowers@superpowers-dev": true` in `~/.claude/settings.json` - -## Integration Test: subagent-driven-development - -### What It Tests - -The integration test verifies the `subagent-driven-development` skill correctly: - -1. **Plan Loading**: Reads the plan once at the beginning -2. **Full Task Text**: Provides complete task descriptions to subagents (doesn't make them read files) -3. **Self-Review**: Ensures subagents perform self-review before reporting -4. **Review Order**: Runs spec compliance review before code quality review -5. **Review Loops**: Uses review loops when issues are found -6. **Independent Verification**: Spec reviewer reads code independently, doesn't trust implementer reports - -### How It Works - -1. **Setup**: Creates a temporary Node.js project with a minimal implementation plan -2. **Execution**: Runs Claude Code in headless mode with the skill -3. **Verification**: Parses the session transcript (`.jsonl` file) to verify: - - Skill tool was invoked - - Subagents were dispatched (Task tool) - - TodoWrite was used for tracking - - Implementation files were created - - Tests pass - - Git commits show proper workflow -4. **Token Analysis**: Shows token usage breakdown by subagent - -### Test Output - -``` -======================================== - Integration Test: subagent-driven-development -======================================== - -Test project: /tmp/tmp.xyz123 +- `tests/brainstorm-server/` — node test suite for the brainstorm server JS code. +- `tests/opencode/` — bash tests for OpenCode plugin loading, bootstrap caching, and tool registration. +- `tests/codex-plugin-sync/` — bash sync verification. +- `tests/claude-code/test-helpers.sh`, `analyze-token-usage.py` — utilities used by remaining bash tests. +- `tests/claude-code/test-subagent-driven-development.sh` — agent-can-describe-SDD test (no drill counterpart; tests description-recall, not behavior). +- `tests/claude-code/test-subagent-driven-development-integration.sh` — extended SDD integration with token analysis (drill covers the YAGNI subset; bash adds commit-count, Claude Code task-tracking, and token telemetry assertions). +- `tests/claude-code/test-worktree-native-preference.sh` — RED-GREEN-REFACTOR validation for worktree skill (drill covers the PRESSURE phase; bash also covers RED/GREEN baselines). +- `tests/explicit-skill-requests/` — Haiku-specific, multi-turn, and skill-name-prompted tests not covered by drill. -=== Verification Tests === - -Test 1: Skill tool invoked... - [PASS] subagent-driven-development skill was invoked - -Test 2: Subagents dispatched... - [PASS] 7 subagents dispatched - -Test 3: Task tracking... - [PASS] TodoWrite used 5 time(s) - -Test 6: Implementation verification... - [PASS] src/math.js created - [PASS] add function exists - [PASS] multiply function exists - [PASS] test/math.test.js created - [PASS] Tests pass - -Test 7: Git commit history... - [PASS] Multiple commits created (3 total) - -Test 8: No extra features added... - [PASS] No extra features added - -========================================= - Token Usage Analysis -========================================= - -Usage Breakdown: ----------------------------------------------------------------------------------------------------- -Agent Description Msgs Input Output Cache Cost ----------------------------------------------------------------------------------------------------- -main Main session (coordinator) 34 27 3,996 1,213,703 $ 4.09 -3380c209 implementing Task 1: Create Add Function 1 2 787 24,989 $ 0.09 -34b00fde implementing Task 2: Create Multiply Function 1 4 644 25,114 $ 0.09 -3801a732 reviewing whether an implementation matches... 1 5 703 25,742 $ 0.09 -4c142934 doing a final code review... 1 6 854 25,319 $ 0.09 -5f017a42 a code reviewer. Review Task 2... 1 6 504 22,949 $ 0.08 -a6b7fbe4 a code reviewer. Review Task 1... 1 6 515 22,534 $ 0.08 -f15837c0 reviewing whether an implementation matches... 1 6 416 22,485 $ 0.07 ----------------------------------------------------------------------------------------------------- - -TOTALS: - Total messages: 41 - Input tokens: 62 - Output tokens: 8,419 - Cache creation tokens: 132,742 - Cache read tokens: 1,382,835 - - Total input (incl cache): 1,515,639 - Total tokens: 1,524,058 - - Estimated cost: $4.67 - (at $3/$15 per M tokens for input/output) - -======================================== - Test Summary -======================================== - -STATUS: PASSED -``` - -## Token Analysis Tool - -### Usage - -Analyze token usage from any Claude Code session: - -```bash -python3 tests/claude-code/analyze-token-usage.py ~/.claude/projects//.jsonl -``` +Run plugin tests via the relevant directory's `run-*.sh` or `npm test`. -### Finding Session Files +## Skill behavior evals -Session transcripts are stored in `~/.claude/projects/` with the working directory path encoded: +Live in `evals/`. Drill is the harness; scenarios live at `evals/scenarios/*.yaml`. See `evals/README.md` for setup. Quick start: ```bash -# Example for /Users/yourname/Documents/GitHub/superpowers/superpowers -SESSION_DIR="$HOME/.claude/projects/-Users-yourname-Documents-GitHub-superpowers-superpowers" - -# Find recent sessions -ls -lt "$SESSION_DIR"/*.jsonl | head -5 -``` - -### What It Shows - -- **Main session usage**: Token usage by the coordinator (you or main Claude instance) -- **Per-subagent breakdown**: Each Task invocation with: - - Agent ID - - Description (extracted from prompt) - - Message count - - Input/output tokens - - Cache usage - - Estimated cost -- **Totals**: Overall token usage and cost estimate - -### Understanding the Output - -- **High cache reads**: Good - means prompt caching is working -- **High input tokens on main**: Expected - coordinator has full context -- **Similar costs per subagent**: Expected - each gets similar task complexity -- **Cost per task**: Typical range is $0.05-$0.15 per subagent depending on task - -## Troubleshooting - -### Skills Not Loading - -**Problem**: Skill not found when running headless tests - -**Solutions**: -1. Ensure you're running FROM the superpowers directory: `cd /path/to/superpowers && tests/...` -2. Check `~/.claude/settings.json` has `"superpowers@superpowers-dev": true` in `enabledPlugins` -3. Verify skill exists in `skills/` directory - -### Permission Errors - -**Problem**: Claude blocked from writing files or accessing directories - -**Solutions**: -1. Use `--permission-mode bypassPermissions` flag -2. Use `--add-dir /path/to/temp/dir` to grant access to test directories -3. Check file permissions on test directories - -### Test Timeouts - -**Problem**: Test takes too long and times out - -**Solutions**: -1. Increase timeout: `timeout 1800 claude ...` (30 minutes) -2. Check for infinite loops in skill logic -3. Review subagent task complexity - -### Session File Not Found - -**Problem**: Can't find session transcript after test run - -**Solutions**: -1. Check the correct project directory in `~/.claude/projects/` -2. Use `find ~/.claude/projects -name "*.jsonl" -mmin -60` to find recent sessions -3. Verify test actually ran (check for errors in test output) - -## Writing New Integration Tests - -### Template - -```bash -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -source "$SCRIPT_DIR/test-helpers.sh" - -# Create test project -TEST_PROJECT=$(create_test_project) -trap "cleanup_test_project $TEST_PROJECT" EXIT - -# Set up test files... -cd "$TEST_PROJECT" - -# Run Claude with skill -PROMPT="Your test prompt here" -cd "$SCRIPT_DIR/../.." && timeout 1800 claude -p "$PROMPT" \ - --allowed-tools=all \ - --add-dir "$TEST_PROJECT" \ - --permission-mode bypassPermissions \ - 2>&1 | tee output.txt - -# Find and analyze session -WORKING_DIR_ESCAPED=$(echo "$SCRIPT_DIR/../.." | sed 's/\\//-/g' | sed 's/^-//') -SESSION_DIR="$HOME/.claude/projects/$WORKING_DIR_ESCAPED" -SESSION_FILE=$(find "$SESSION_DIR" -name "*.jsonl" -type f -mmin -60 | sort -r | head -1) - -# Verify behavior by parsing session transcript -if grep -q '"name":"Skill".*"skill":"your-skill-name"' "$SESSION_FILE"; then - echo "[PASS] Skill was invoked" -fi - -# Show token analysis -python3 "$SCRIPT_DIR/analyze-token-usage.py" "$SESSION_FILE" -``` - -### Best Practices - -1. **Always cleanup**: Use trap to cleanup temp directories -2. **Parse transcripts**: Don't grep user-facing output - parse the `.jsonl` session file -3. **Grant permissions**: Use `--permission-mode bypassPermissions` and `--add-dir` -4. **Run from plugin dir**: Skills only load when running from the superpowers directory -5. **Show token usage**: Always include token analysis for cost visibility -6. **Test real behavior**: Verify actual files created, tests passing, commits made - -## Session Transcript Format - -Session transcripts are JSONL (JSON Lines) files where each line is a JSON object representing a message or tool result. - -### Key Fields - -```json -{ - "type": "assistant", - "message": { - "content": [...], - "usage": { - "input_tokens": 27, - "output_tokens": 3996, - "cache_read_input_tokens": 1213703 - } - } -} -``` - -### Tool Results - -```json -{ - "type": "user", - "toolUseResult": { - "agentId": "3380c209", - "usage": { - "input_tokens": 2, - "output_tokens": 787, - "cache_read_input_tokens": 24989 - }, - "prompt": "You are implementing Task 1...", - "content": [{"type": "text", "text": "..."}] - } -} +cd evals +uv sync --extra dev +export ANTHROPIC_API_KEY=sk-... +uv run drill run triggering-test-driven-development -b claude ``` -The `agentId` field links to subagent sessions, and the `usage` field contains token usage for that specific subagent invocation. +Drill scenarios are slow (3-30+ minutes each) and run real LLM sessions. They are not part of CI today; the natural follow-up is a tiered model (fast subset on PR, full sweep nightly + on-demand). diff --git a/evals b/evals new file mode 160000 index 0000000000..e2b37138c8 --- /dev/null +++ b/evals @@ -0,0 +1 @@ +Subproject commit e2b37138c8c636561febf4b18a1f9e401874ca69 diff --git a/hooks/hooks-codex.json b/hooks/hooks-codex.json new file mode 100644 index 0000000000..5c357fccf6 --- /dev/null +++ b/hooks/hooks-codex.json @@ -0,0 +1,16 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume|clear", + "hooks": [ + { + "type": "command", + "command": "\"${PLUGIN_ROOT}/hooks/run-hook.cmd\" session-start-codex", + "async": false + } + ] + } + ] + } +} diff --git a/hooks/session-start b/hooks/session-start index 2460429503..93a6bc2c6b 100755 --- a/hooks/session-start +++ b/hooks/session-start @@ -7,13 +7,6 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PLUGIN_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -# Check if legacy skills directory exists and build warning -warning_message="" -legacy_skills_dir="${HOME}/.config/superpowers/skills" -if [ -d "$legacy_skills_dir" ]; then - warning_message="\n\nIN YOUR FIRST REPLY AFTER SEEING THIS MESSAGE YOU MUST TELL THE USER:⚠️ **WARNING:** Superpowers now uses Claude Code's skills system. Custom skills in ~/.config/superpowers/skills will not be read. Move custom skills to ~/.claude/skills instead. To make this message go away, remove ~/.config/superpowers/skills" -fi - # Read using-superpowers content using_superpowers_content=$(cat "${PLUGIN_ROOT}/skills/using-superpowers/SKILL.md" 2>&1 || echo "Error reading using-superpowers skill") @@ -31,8 +24,7 @@ escape_for_json() { } using_superpowers_escaped=$(escape_for_json "$using_superpowers_content") -warning_escaped=$(escape_for_json "$warning_message") -session_context="\nYou have superpowers.\n\n**Below is the full content of your 'superpowers:using-superpowers' skill - your introduction to using skills. For all other skills, use the 'Skill' tool:**\n\n${using_superpowers_escaped}\n\n${warning_escaped}\n" +session_context="\nYou have superpowers.\n\n**Below is the full content of your 'superpowers:using-superpowers' skill - your introduction to using skills. For all other skills, use the 'Skill' tool:**\n\n${using_superpowers_escaped}\n" # Output context injection as JSON. # Cursor hooks expect additional_context (snake_case). @@ -45,13 +37,13 @@ session_context="\nYou have superpowers.\n\n**Below is the # See: https://github.com/obra/superpowers/issues/571 if [ -n "${CURSOR_PLUGIN_ROOT:-}" ]; then # Cursor sets CURSOR_PLUGIN_ROOT (may also set CLAUDE_PLUGIN_ROOT) - printf '{\n "additional_context": "%s"\n}\n' "$session_context" + printf '{\n "additional_context": "%s"\n}\n' "$session_context" | cat elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -z "${COPILOT_CLI:-}" ]; then # Claude Code sets CLAUDE_PLUGIN_ROOT without COPILOT_CLI - printf '{\n "hookSpecificOutput": {\n "hookEventName": "SessionStart",\n "additionalContext": "%s"\n }\n}\n' "$session_context" + printf '{\n "hookSpecificOutput": {\n "hookEventName": "SessionStart",\n "additionalContext": "%s"\n }\n}\n' "$session_context" | cat else # Copilot CLI (sets COPILOT_CLI=1) or unknown platform — SDK standard format - printf '{\n "additionalContext": "%s"\n}\n' "$session_context" + printf '{\n "additionalContext": "%s"\n}\n' "$session_context" | cat fi exit 0 diff --git a/hooks/session-start-codex b/hooks/session-start-codex new file mode 100755 index 0000000000..f25ea0846e --- /dev/null +++ b/hooks/session-start-codex @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Codex SessionStart hook for superpowers plugin + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PLUGIN_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +using_superpowers_content=$(cat "${PLUGIN_ROOT}/skills/using-superpowers/SKILL.md" 2>&1 || echo "Error reading using-superpowers skill") + +escape_for_json() { + local s="$1" + s="${s//\\/\\\\}" + s="${s//\"/\\\"}" + s="${s//$'\n'/\\n}" + s="${s//$'\r'/\\r}" + s="${s//$'\t'/\\t}" + printf '%s' "$s" +} + +using_superpowers_escaped=$(escape_for_json "$using_superpowers_content") +session_context="\nYou have superpowers.\n\n**Below is the full content of your 'superpowers:using-superpowers' skill - your introduction to using skills. For all other skills, follow the Codex skill-loading instructions in that skill:**\n\n${using_superpowers_escaped}\n" + +printf '{\n "hookSpecificOutput": {\n "hookEventName": "SessionStart",\n "additionalContext": "%s"\n }\n}\n' "$session_context" | cat + +exit 0 diff --git a/package.json b/package.json index 2b8146635e..54c7bdfa8a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,23 @@ { "name": "superpowers", "version": "5.1.0", + "description": "Superpowers skills and runtime bootstrap for coding agents", "type": "module", - "main": ".opencode/plugins/superpowers.js" + "main": ".opencode/plugins/superpowers.js", + "keywords": [ + "pi-package", + "skills", + "tdd", + "debugging", + "collaboration", + "workflow" + ], + "pi": { + "extensions": [ + "./.pi/extensions/superpowers.ts" + ], + "skills": [ + "./skills" + ] + } } diff --git a/scripts/sync-to-codex-plugin.sh b/scripts/sync-to-codex-plugin.sh index fc0a8e85d0..f490836582 100755 --- a/scripts/sync-to-codex-plugin.sh +++ b/scripts/sync-to-codex-plugin.sh @@ -69,7 +69,7 @@ EXCLUDES=( # Directories not shipped by canonical Codex plugins "/commands/" "/docs/" - "/hooks/" + "/evals/" "/lib/" "/scripts/" "/tests/" @@ -415,26 +415,73 @@ fi git add "$DEST_REL" +vendor_notice_for_pr_body() { + local provenance_glob="$DEST"/skills/*/scripts/vendor/*.provenance.json + + if ! compgen -G "$provenance_glob" > /dev/null; then + return 0 + fi + + command -v python3 >/dev/null || die "python3 not found in PATH" + python3 - "$DEST" <<'PY' +import glob +import json +import os +import sys + +dest = sys.argv[1] +provenance_files = sorted(glob.glob(os.path.join(dest, "skills", "*", "scripts", "vendor", "*.provenance.json"))) +if not provenance_files: + raise SystemExit(0) + +print() +print() +print("Vendored third-party code included in this sync:") +for provenance_file in provenance_files: + with open(provenance_file, "r", encoding="utf-8") as fh: + provenance = json.load(fh) + + rel_provenance = os.path.relpath(provenance_file, dest) + rel_vendor_dir = os.path.dirname(rel_provenance) + basename = os.path.basename(provenance_file) + suffix = ".provenance.json" + if basename.endswith(suffix): + basename = basename[:-len(suffix)] + local_path = provenance.get("localPath") or os.path.join(rel_vendor_dir, f"{basename}.js") + notice_path = os.path.join(rel_vendor_dir, "THIRD_PARTY_NOTICES.md") + name = provenance.get("name", "unknown") + version = provenance.get("version", "unknown") + approval = provenance.get("approvalArtifact", "not recorded") + sha256 = provenance.get("sha256", "not recorded") + + print(f"- `{local_path}`: {name} {version}") + print(f" - Approval artifact: {approval}") + print(f" - License notice: `{notice_path}`") + print(f" - Provenance: `{rel_provenance}`") + print(f" - SHA256: `{sha256}`") +PY +} + if [[ $BOOTSTRAP -eq 1 ]]; then COMMIT_TITLE="bootstrap superpowers v$UPSTREAM_VERSION from upstream main @ $UPSTREAM_SHORT" PR_BODY="Initial bootstrap of the superpowers plugin from upstream \`main\` @ \`$UPSTREAM_SHORT\` (v$UPSTREAM_VERSION). -Creates \`plugins/superpowers/\` by copying the tracked plugin files from upstream, including \`.codex-plugin/plugin.json\` and \`assets/\`. +Creates \`plugins/superpowers/\` by copying the tracked plugin files from upstream, including \`.codex-plugin/plugin.json\`, \`assets/\`, and \`hooks/\`. Run via: \`scripts/sync-to-codex-plugin.sh --bootstrap\` Upstream commit: https://github.com/obra/superpowers/commit/$UPSTREAM_SHA -This is a one-time bootstrap. Subsequent syncs will be normal (non-bootstrap) runs using the same tracked upstream plugin files." +This is a one-time bootstrap. Subsequent syncs will be normal (non-bootstrap) runs using the same tracked upstream plugin files.$(vendor_notice_for_pr_body)" else COMMIT_TITLE="sync superpowers v$UPSTREAM_VERSION from upstream main @ $UPSTREAM_SHORT" PR_BODY="Automated sync from superpowers upstream \`main\` @ \`$UPSTREAM_SHORT\` (v$UPSTREAM_VERSION). -Copies the tracked plugin files from upstream, including the committed Codex manifest and assets. +Copies the tracked plugin files from upstream, including the committed Codex manifest, assets, and hooks. Run via: \`scripts/sync-to-codex-plugin.sh\` Upstream commit: https://github.com/obra/superpowers/commit/$UPSTREAM_SHA -Running the sync tool again against the same upstream SHA should produce a PR with an identical diff — use that to verify the tool is behaving." +Running the sync tool again against the same upstream SHA should produce a PR with an identical diff — use that to verify the tool is behaving.$(vendor_notice_for_pr_body)" fi git commit --quiet -m "$COMMIT_TITLE diff --git a/skills/brainstorming/scripts/frame-template.html b/skills/brainstorming/scripts/frame-template.html index dcfe01817e..7a3c8ec81c 100644 --- a/skills/brainstorming/scripts/frame-template.html +++ b/skills/brainstorming/scripts/frame-template.html @@ -13,7 +13,7 @@ * - Scrollable main content area * - CSS helpers for common UI patterns * - * Content is injected via placeholder comment in #claude-content. + * Content is injected via placeholder comment in #frame-content. */ * { box-sizing: border-box; margin: 0; padding: 0; } @@ -77,7 +77,7 @@ .header .status::before { content: ''; width: 6px; height: 6px; background: var(--success); border-radius: 50%; } .main { flex: 1; overflow-y: auto; } - #claude-content { padding: 2rem; min-height: 100%; } + #frame-content { padding: 2rem; min-height: 100%; } .indicator-bar { background: var(--bg-secondary); @@ -193,6 +193,7 @@ .mock-button { background: var(--accent); color: white; border: none; padding: 0.5rem 1rem; border-radius: 6px; font-size: 0.85rem; } .mock-input { background: var(--bg-primary); border: 1px solid var(--border); border-radius: 6px; padding: 0.5rem; width: 100%; } +
@@ -201,13 +202,13 @@

-
+
- Click an option above, then return to the terminal + Interact with the mockup, then return to the terminal
diff --git a/skills/brainstorming/scripts/helper.js b/skills/brainstorming/scripts/helper.js index 111f97f597..c804e752ab 100644 --- a/skills/brainstorming/scripts/helper.js +++ b/skills/brainstorming/scripts/helper.js @@ -51,7 +51,7 @@ const container = target.closest('.options') || target.closest('.cards'); const selected = container ? container.querySelectorAll('.selected') : []; if (selected.length === 0) { - indicator.textContent = 'Click an option above, then return to the terminal'; + indicator.textContent = 'Interact with the mockup, then return to the terminal'; } else if (selected.length === 1) { const label = selected[0].querySelector('h3, .content h3, .card-body h3')?.textContent?.trim() || selected[0].dataset.choice; indicator.innerHTML = '' + label + ' selected — return to terminal to continue'; diff --git a/skills/brainstorming/scripts/server.cjs b/skills/brainstorming/scripts/server.cjs index 562c17f893..9af74adac4 100644 --- a/skills/brainstorming/scripts/server.cjs +++ b/skills/brainstorming/scripts/server.cjs @@ -101,6 +101,26 @@ h1 { color: #333; } p { color: #666; } const frameTemplate = fs.readFileSync(path.join(__dirname, 'frame-template.html'), 'utf-8'); const helperScript = fs.readFileSync(path.join(__dirname, 'helper.js'), 'utf-8'); const helperInjection = ''; +const ALPINE_VENDOR_PATH = path.join(__dirname, 'vendor', 'alpine.js'); + +function loadVendorFile(filePath, name) { + try { + return fs.readFileSync(filePath); + } catch (error) { + throw new Error( + `Failed to load vendored ${name} at ${filePath}; ` + + 'run the refresh command in skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md. ' + + error.message + ); + } +} + +const VENDOR_FILES = new Map([ + ['/vendor/alpine.js', { + content: loadVendorFile(ALPINE_VENDOR_PATH, 'Alpine'), + contentType: 'application/javascript; charset=utf-8' + }] +]); // ========== Helper Functions ========== @@ -124,11 +144,30 @@ function getNewestScreen() { return files.length > 0 ? files[0].path : null; } +function parseRequestUrl(req) { + // Vendor routing depends on URL normalization before exact pathname allowlist checks. + return new URL(req.url, 'http://localhost'); +} + +function serveVendorFile(requestUrl, res) { + const vendorFile = VENDOR_FILES.get(requestUrl.pathname); + if (!vendorFile) { + res.writeHead(404); + res.end('Not found'); + return; + } + + res.writeHead(200, { 'Content-Type': vendorFile.contentType }); + res.end(vendorFile.content); +} + // ========== HTTP Request Handler ========== function handleRequest(req, res) { touchActivity(); - if (req.method === 'GET' && req.url === '/') { + const requestUrl = parseRequestUrl(req); + + if (req.method === 'GET' && requestUrl.pathname === '/') { const screenFile = getNewestScreen(); let html = screenFile ? (raw => isFullDocument(raw) ? raw : wrapInFrame(raw))(fs.readFileSync(screenFile, 'utf-8')) @@ -142,8 +181,10 @@ function handleRequest(req, res) { res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); res.end(html); - } else if (req.method === 'GET' && req.url.startsWith('/files/')) { - const fileName = req.url.slice(7); + } else if (req.method === 'GET' && requestUrl.pathname.startsWith('/vendor/')) { + serveVendorFile(requestUrl, res); + } else if (req.method === 'GET' && requestUrl.pathname.startsWith('/files/')) { + const fileName = requestUrl.pathname.slice(7); const filePath = path.join(CONTENT_DIR, path.basename(fileName)); if (!fs.existsSync(filePath)) { res.writeHead(404); diff --git a/skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md b/skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000000..a53a2effa7 --- /dev/null +++ b/skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md @@ -0,0 +1,48 @@ +# Third-Party Notices + +## Alpine.js + +- Package: `alpinejs` +- Version: `3.15.12` +- Source: `https://registry.npmjs.org/alpinejs/-/alpinejs-3.15.12.tgz` +- Vendored file: `package/dist/cdn.min.js` +- Local path: `skills/brainstorming/scripts/vendor/alpine.js` +- SHA256: `57b37d7cae9a27d965fdae4adcc844245dfdc407e655aee85dcfff3a08036a3f` + +Refresh command: + +```bash +cd "$(git rev-parse --show-toplevel)" +tmpdir="$(mktemp -d)" +curl -fsSL https://registry.npmjs.org/alpinejs/-/alpinejs-3.15.12.tgz -o "$tmpdir/alpinejs-3.15.12.tgz" +tar -xzf "$tmpdir/alpinejs-3.15.12.tgz" -C "$tmpdir" package/dist/cdn.min.js +cp "$tmpdir/package/dist/cdn.min.js" skills/brainstorming/scripts/vendor/alpine.js +shasum -a 256 skills/brainstorming/scripts/vendor/alpine.js +rm -rf "$tmpdir" +``` + +License: + +```text +MIT License + +Copyright © 2019-2025 Caleb Porzio and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` diff --git a/skills/brainstorming/scripts/vendor/alpine.js b/skills/brainstorming/scripts/vendor/alpine.js new file mode 100644 index 0000000000..ab371ef48a --- /dev/null +++ b/skills/brainstorming/scripts/vendor/alpine.js @@ -0,0 +1,5 @@ +(()=>{var ee=!1,re=!1,W=[],ne=-1,ie=!1;function Ve(t){Dn(t)}function Ue(){ie=!0}function qe(){ie=!1,We()}function Dn(t){W.includes(t)||W.push(t),We()}function Ke(t){let e=W.indexOf(t);e!==-1&&e>ne&&W.splice(e,1)}function We(){if(!re&&!ee){if(ie)return;ee=!0,queueMicrotask(In)}}function In(){ee=!1,re=!0;for(let t=0;tt.effect(e,{scheduler:r=>{oe?Ve(r):r()}}),se=t.raw}function ae(t){R=t}function Ye(t){let e=()=>{};return[n=>{let i=R(n);return t._x_effects||(t._x_effects=new Set,t._x_runEffects=()=>{t._x_effects.forEach(o=>o())}),t._x_effects.add(i),e=()=>{i!==void 0&&(t._x_effects.delete(i),j(i))},i},()=>{e()}]}function St(t,e){let r=!0,n,i,o=R(()=>{let s=t(),a=JSON.stringify(s);if(!r&&(typeof s=="object"||s!==n)){let c=typeof n=="object"?JSON.parse(i):n;queueMicrotask(()=>{e(s,c)})}n=s,i=a,r=!1});return()=>j(o)}async function Xe(t){Ue();try{await t(),await Promise.resolve()}finally{qe()}}var Ze=[],Qe=[],tr=[];function er(t){tr.push(t)}function et(t,e){typeof e=="function"?(t._x_cleanups||(t._x_cleanups=[]),t._x_cleanups.push(e)):(e=t,Qe.push(e))}function At(t){Ze.push(t)}function Ot(t,e,r){t._x_attributeCleanups||(t._x_attributeCleanups={}),t._x_attributeCleanups[e]||(t._x_attributeCleanups[e]=[]),t._x_attributeCleanups[e].push(r)}function ce(t,e){t._x_attributeCleanups&&Object.entries(t._x_attributeCleanups).forEach(([r,n])=>{(e===void 0||e.includes(r))&&(n.forEach(i=>i()),delete t._x_attributeCleanups[r])})}function rr(t){for(t._x_effects?.forEach(Ke);t._x_cleanups?.length;)t._x_cleanups.pop()()}var le=new MutationObserver(pe),ue=!1;function ut(){le.observe(document,{subtree:!0,childList:!0,attributes:!0,attributeOldValue:!0}),ue=!0}function fe(){kn(),le.disconnect(),ue=!1}var lt=[];function kn(){let t=le.takeRecords();lt.push(()=>t.length>0&&pe(t));let e=lt.length;queueMicrotask(()=>{if(lt.length===e)for(;lt.length>0;)lt.shift()()})}function m(t){if(!ue)return t();fe();let e=t();return ut(),e}var de=!1,vt=[];function nr(){de=!0}function ir(){de=!1,pe(vt),vt=[]}function pe(t){if(de){vt=vt.concat(t);return}let e=[],r=new Set,n=new Map,i=new Map;for(let o=0;o{s.nodeType===1&&s._x_marker&&r.add(s)}),t[o].addedNodes.forEach(s=>{if(s.nodeType===1){if(r.has(s)){r.delete(s);return}s._x_marker||e.push(s)}})),t[o].type==="attributes")){let s=t[o].target,a=t[o].attributeName,c=t[o].oldValue,l=()=>{n.has(s)||n.set(s,[]),n.get(s).push({name:a,value:s.getAttribute(a)})},u=()=>{i.has(s)||i.set(s,[]),i.get(s).push(a)};s.hasAttribute(a)&&c===null?l():s.hasAttribute(a)?(u(),l()):u()}i.forEach((o,s)=>{ce(s,o)}),n.forEach((o,s)=>{Ze.forEach(a=>a(s,o))});for(let o of r)e.some(s=>s.contains(o))||Qe.forEach(s=>s(o));for(let o of e)o.isConnected&&tr.forEach(s=>s(o));e=null,r=null,n=null,i=null}function Ct(t){return P(F(t))}function N(t,e,r){return t._x_dataStack=[e,...F(r||t)],()=>{t._x_dataStack=t._x_dataStack.filter(n=>n!==e)}}function F(t){return t._x_dataStack?t._x_dataStack:typeof ShadowRoot=="function"&&t instanceof ShadowRoot?F(t.host):t.parentNode?F(t.parentNode):[]}function P(t){return new Proxy({objects:t},$n)}function or(t,e){return t===null||t===Object.prototype?null:Object.prototype.hasOwnProperty.call(t,e)?t:or(Object.getPrototypeOf(t),e)}var $n={ownKeys({objects:t}){return Array.from(new Set(t.flatMap(e=>Object.keys(e))))},has({objects:t},e){return e==Symbol.unscopables?!1:t.some(r=>Object.prototype.hasOwnProperty.call(r,e)||Reflect.has(r,e))},get({objects:t},e,r){return e=="toJSON"?Ln:Reflect.get(t.find(n=>Reflect.has(n,e))||{},e,r)},set({objects:t},e,r,n){let i;for(let s of t)if(i=or(s,e),i)break;i||(i=t[t.length-1]);let o=Object.getOwnPropertyDescriptor(i,e);return o?.set&&o?.get?o.set.call(n,r)||!0:Reflect.set(i,e,r)}};function Ln(){return Reflect.ownKeys(this).reduce((e,r)=>(e[r]=Reflect.get(this,r),e),{})}function rt(t){let e=n=>typeof n=="object"&&!Array.isArray(n)&&n!==null,r=(n,i="")=>{Object.entries(Object.getOwnPropertyDescriptors(n)).forEach(([o,{value:s,enumerable:a}])=>{if(a===!1||s===void 0||typeof s=="object"&&s!==null&&s.__v_skip)return;let c=i===""?o:`${i}.${o}`;typeof s=="object"&&s!==null&&s._x_interceptor?n[o]=s.initialize(t,c,o):e(s)&&s!==n&&!(s instanceof Element)&&r(s,c)})};return r(t)}function Tt(t,e=()=>{}){let r={initialValue:void 0,_x_interceptor:!0,initialize(n,i,o){return t(this.initialValue,()=>jn(n,i),s=>me(n,i,s),i,o)}};return e(r),n=>{if(typeof n=="object"&&n!==null&&n._x_interceptor){let i=r.initialize.bind(r);r.initialize=(o,s,a)=>{let c=n.initialize(o,s,a);return r.initialValue=c,i(o,s,a)}}else r.initialValue=n;return r}}function jn(t,e){return e.split(".").reduce((r,n)=>r[n],t)}function me(t,e,r){if(typeof e=="string"&&(e=e.split(".")),e.length===1)t[e[0]]=r;else{if(e.length===0)throw error;return t[e[0]]||(t[e[0]]={}),me(t[e[0]],e.slice(1),r)}}var sr={};function x(t,e){sr[t]=e}function H(t,e){let r=Fn(e);return Object.entries(sr).forEach(([n,i])=>{Object.defineProperty(t,`$${n}`,{get(){return i(e,r)},enumerable:!1})}),t}function Fn(t){let[e,r]=he(t),n={interceptor:Tt,...e};return et(t,r),n}function ar(t,e,r,...n){try{return r(...n)}catch(i){nt(i,t,e)}}function nt(...t){return cr(...t)}var cr=Bn;function lr(t){cr=t}function Bn(t,e,r=void 0){t=Object.assign(t??{message:"No error message given."},{el:e,expression:r}),console.warn(`Alpine Expression Error: ${t.message} + +${r?'Expression: "'+r+`" + +`:""}`,e),setTimeout(()=>{throw t},0)}var it=!0;function Mt(t){let e=it;it=!1;let r=t();return it=e,r}function T(t,e,r={}){let n;return _(t,e)(i=>n=i,r),n}function _(...t){return ur(...t)}var ur=()=>{};function fr(t){ur=t}var dr;function pr(t){dr=t}function mr(t,e){let r={};H(r,t);let n=[r,...F(t)],i=typeof e=="function"?zn(n,e):Vn(n,e,t);return ar.bind(null,t,e,i)}function zn(t,e){return(r=()=>{},{scope:n={},params:i=[],context:o}={})=>{if(!it){ft(r,e,P([n,...t]),i);return}let s=e.apply(P([n,...t]),i);ft(r,s)}}var _e={};function Hn(t,e){if(_e[t])return _e[t];let r=Object.getPrototypeOf(async function(){}).constructor,n=/^[\n\s]*if.*\(.*\)/.test(t.trim())||/^(let|const)\s/.test(t.trim())?`(async()=>{ ${t} })()`:t,o=(()=>{try{let s=new r(["__self","scope"],`with (scope) { __self.result = ${n} }; __self.finished = true; return __self.result;`);return Object.defineProperty(s,"name",{value:`[Alpine] ${t}`}),s}catch(s){return nt(s,e,t),Promise.resolve()}})();return _e[t]=o,o}function Vn(t,e,r){let n=Hn(e,r);return(i=()=>{},{scope:o={},params:s=[],context:a}={})=>{n.result=void 0,n.finished=!1;let c=P([o,...t]);if(typeof n=="function"){let l=n.call(a,n,c).catch(u=>nt(u,r,e));n.finished?(ft(i,n.result,c,s,r),n.result=void 0):l.then(u=>{ft(i,u,c,s,r)}).catch(u=>nt(u,r,e)).finally(()=>n.result=void 0)}}}function ft(t,e,r,n,i){if(it&&typeof e=="function"){let o=e.apply(r,n);o instanceof Promise?o.then(s=>ft(t,s,r,n)).catch(s=>nt(s,i,e)):t(o)}else typeof e=="object"&&e instanceof Promise?e.then(o=>t(o)):t(e)}function hr(...t){return dr(...t)}function _r(t,e,r={}){let n={};H(n,t);let i=[n,...F(t)],o=P([r.scope??{},...i]),s=r.params??[];if(e.includes("await")){let a=Object.getPrototypeOf(async function(){}).constructor,c=/^[\n\s]*if.*\(.*\)/.test(e.trim())||/^(let|const)\s/.test(e.trim())?`(async()=>{ ${e} })()`:e;return new a(["scope"],`with (scope) { let __result = ${c}; return __result }`).call(r.context,o)}else{let a=/^[\n\s]*if.*\(.*\)/.test(e.trim())||/^(let|const)\s/.test(e.trim())?`(()=>{ ${e} })()`:e,l=new Function(["scope"],`with (scope) { let __result = ${a}; return __result }`).call(r.context,o);return typeof l=="function"&&it?l.apply(o,s):l}}var ye="x-";function O(t=""){return ye+t}function gr(t){ye=t}var Rt={};function p(t,e){return Rt[t]=e,{before(r){if(!Rt[r]){console.warn(String.raw`Cannot find directive \`${r}\`. \`${t}\` will use the default order of execution`);return}let n=G.indexOf(r);G.splice(n>=0?n:G.indexOf("DEFAULT"),0,t)}}}function xr(t){return Object.keys(Rt).includes(t)}function pt(t,e,r){if(e=Array.from(e),t._x_virtualDirectives){let o=Object.entries(t._x_virtualDirectives).map(([a,c])=>({name:a,value:c})),s=be(o);o=o.map(a=>s.find(c=>c.name===a.name)?{name:`x-bind:${a.name}`,value:`"${a.value}"`}:a),e=e.concat(o)}let n={};return e.map(wr((o,s)=>n[o]=s)).filter(Sr).map(qn(n,r)).sort(Kn).map(o=>Un(t,o))}function be(t){return Array.from(t).map(wr()).filter(e=>!Sr(e))}var ge=!1,dt=new Map,yr=Symbol();function br(t){ge=!0;let e=Symbol();yr=e,dt.set(e,[]);let r=()=>{for(;dt.get(e).length;)dt.get(e).shift()();dt.delete(e)},n=()=>{ge=!1,r()};t(r),n()}function he(t){let e=[],r=a=>e.push(a),[n,i]=Ye(t);return e.push(i),[{Alpine:B,effect:n,cleanup:r,evaluateLater:_.bind(_,t),evaluate:T.bind(T,t)},()=>e.forEach(a=>a())]}function Un(t,e){let r=()=>{},n=Rt[e.type]||r,[i,o]=he(t);Ot(t,e.original,o);let s=()=>{t._x_ignore||t._x_ignoreSelf||(n.inline&&n.inline(t,e,i),n=n.bind(n,t,e,i),ge?dt.get(yr).push(n):n())};return s.runCleanups=o,s}var Nt=(t,e)=>({name:r,value:n})=>(r.startsWith(t)&&(r=r.replace(t,e)),{name:r,value:n}),Pt=t=>t;function wr(t=()=>{}){return({name:e,value:r})=>{let{name:n,value:i}=Er.reduce((o,s)=>s(o),{name:e,value:r});return n!==e&&t(n,e),{name:n,value:i}}}var Er=[];function ot(t){Er.push(t)}function Sr({name:t}){return vr().test(t)}var vr=()=>new RegExp(`^${ye}([^:^.]+)\\b`);function qn(t,e){return({name:r,value:n})=>{r===n&&(n="");let i=r.match(vr()),o=r.match(/:([a-zA-Z0-9\-_:]+)/),s=r.match(/\.[^.\]]+(?=[^\]]*$)/g)||[],a=e||t[r]||r;return{type:i?i[1]:null,value:o?o[1]:null,modifiers:s.map(c=>c.replace(".","")),expression:n,original:a}}}var xe="DEFAULT",G=["ignore","ref","id","data","anchor","bind","init","for","model","modelable","transition","show","if",xe,"teleport"];function Kn(t,e){let r=G.indexOf(t.type)===-1?xe:t.type,n=G.indexOf(e.type)===-1?xe:e.type;return G.indexOf(r)-G.indexOf(n)}function J(t,e,r={},n={}){return t.dispatchEvent(new CustomEvent(e,{detail:r,bubbles:!0,composed:!0,cancelable:!0,...n}))}function D(t,e){if(typeof ShadowRoot=="function"&&t instanceof ShadowRoot){Array.from(t.children).forEach(i=>D(i,e));return}let r=!1;if(e(t,()=>r=!0),r)return;let n=t.firstElementChild;for(;n;)D(n,e,!1),n=n.nextElementSibling}function E(t,...e){console.warn(`Alpine Warning: ${t}`,...e)}var Ar=!1;function Or(){Ar&&E("Alpine has already been initialized on this page. Calling Alpine.start() more than once can cause problems."),Ar=!0,document.body||E("Unable to initialize. Trying to load Alpine before `` is available. Did you forget to add `defer` in Alpine's `'), 'Fragment should load Alpine'); + assert(res.body.includes('Interact with the mockup, then return to the terminal'), 'Frame copy should be neutral'); + }); + + await test('preserves Alpine attributes in frame-wrapped fragments', async () => { + const fragment = '
Details
'; + fs.writeFileSync(path.join(CONTENT_DIR, 'alpine-fragment.html'), fragment); + await sleep(300); + + const res = await fetch(`http://localhost:${TEST_PORT}/`); + assert(res.body.includes('x-data="{ open: false }"'), 'Should preserve x-data'); + assert(res.body.includes('@click="open = !open"'), 'Should preserve @click'); + assert(res.body.includes('x-show="open"'), 'Should preserve x-show'); + assert(res.body.includes('/vendor/alpine.js'), 'Should include Alpine script'); }); await test('serves newest file by mtime', async () => { @@ -184,6 +264,48 @@ async function runTests() { assert.strictEqual(res.status, 404); }); + await test('serves files by pathname when query string is present', async () => { + fs.writeFileSync(path.join(CONTENT_DIR, 'asset.png'), 'image-bytes'); + const res = await fetch(`http://localhost:${TEST_PORT}/files/asset.png?v=1`); + assert.strictEqual(res.status, 200); + assert.strictEqual(res.body, 'image-bytes'); + }); + + await test('serves vendored Alpine from exact vendor route', async () => { + const res = await fetch(`http://localhost:${TEST_PORT}/vendor/alpine.js`); + const provenance = JSON.parse(fs.readFileSync(ALPINE_PROVENANCE_PATH, 'utf-8')); + assert.strictEqual(res.status, 200); + assert(res.headers['content-type'].includes('application/javascript'), 'Should be JavaScript'); + assert.strictEqual( + crypto.createHash('sha256').update(res.body).digest('hex'), + provenance.sha256, + 'Should serve the pinned Alpine artifact' + ); + }); + + await test('serves vendored Alpine when query string is present', async () => { + const res = await fetch(`http://localhost:${TEST_PORT}/vendor/alpine.js?v=3.15.12`); + assert.strictEqual(res.status, 200); + assert(res.body.includes('Alpine'), 'Should ignore query string for exact vendor pathname'); + }); + + await test('exact-match vendor route rejects non-allowlisted pathnames', async () => { + const paths = [ + '/vendor/unknown.js', + '/vendor/alpine.js/extra', + '/vendor/%2e%2e/alpine.js', + '/vendor/%2E%2E/alpine.js' + ]; + + for (const requestPath of paths) { + const res = await fetch(`http://localhost:${TEST_PORT}${requestPath}`); + assert.strictEqual(res.status, 404, `${requestPath} should 404`); + } + + const dotSegmentRes = await rawHttpRequest('/vendor/../alpine.js'); + assert.strictEqual(dotSegmentRes.status, 404, 'raw dot-segment vendor path should 404'); + }); + // ========== WebSocket Communication ========== console.log('\n--- WebSocket Communication ---'); @@ -396,6 +518,15 @@ async function runTests() { return Promise.resolve(); }); + await test('helper.js keeps indicator fallback copy neutral', () => { + const helperContent = fs.readFileSync( + path.join(__dirname, '../../skills/brainstorming/scripts/helper.js'), 'utf-8' + ); + assert(helperContent.includes('Interact with the mockup, then return to the terminal'), 'Should use neutral fallback copy'); + assert(!helperContent.includes('Click an option above, then return to the terminal'), 'Should not reset to selection-first copy'); + return Promise.resolve(); + }); + // ========== Frame Template ========== console.log('\n--- Frame Template Verification ---'); @@ -406,7 +537,7 @@ async function runTests() { assert(template.includes('indicator-bar'), 'Should have indicator bar'); assert(template.includes('indicator-text'), 'Should have indicator text'); assert(template.includes(''), 'Should have content placeholder'); - assert(template.includes('claude-content'), 'Should have content container'); + assert(template.includes('frame-content'), 'Should have content container'); return Promise.resolve(); }); diff --git a/tests/brainstorm-server/windows-lifecycle.test.sh b/tests/brainstorm-server/windows-lifecycle.test.sh index b15a588d4e..d86781f389 100755 --- a/tests/brainstorm-server/windows-lifecycle.test.sh +++ b/tests/brainstorm-server/windows-lifecycle.test.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash # Windows lifecycle tests for the brainstorm server. # -# Verifies that the brainstorm server survives the 60-second lifecycle -# check on Windows, where OWNER_PID monitoring is disabled because the -# MSYS2 PID namespace is invisible to Node.js. +# Verifies brainstorm server lifecycle behavior, including: +# - Windows/MSYS2 foreground mode and empty OWNER_PID handling +# - Server survival past the 60-second lifecycle check window +# - Dead-at-startup OWNER_PID validation (logged, monitoring disabled) +# - Clean stop-server.sh shutdown # # Requirements: # - Node.js in PATH @@ -20,7 +22,7 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="${SUPERPOWERS_ROOT:-$(cd "$SCRIPT_DIR/../.." && pwd)}" START_SCRIPT="$REPO_ROOT/skills/brainstorming/scripts/start-server.sh" STOP_SCRIPT="$REPO_ROOT/skills/brainstorming/scripts/stop-server.sh" -SERVER_JS="$REPO_ROOT/skills/brainstorming/scripts/server.js" +SERVER_SCRIPT="$REPO_ROOT/skills/brainstorming/scripts/server.cjs" TEST_DIR="${TMPDIR:-/tmp}/brainstorm-win-test-$$" @@ -64,7 +66,7 @@ skip() { wait_for_server_info() { local dir="$1" for _ in $(seq 1 50); do - if [[ -f "$dir/.server-info" ]]; then + if [[ -f "$dir/state/server-info" ]]; then return 0 fi sleep 0.1 @@ -73,9 +75,9 @@ wait_for_server_info() { } get_port_from_info() { - # Read the port from .server-info. Use grep/sed instead of Node.js + # Read the port from state/server-info. Use grep/sed instead of Node.js # to avoid MSYS2-to-Windows path translation issues. - grep -o '"port":[0-9]*' "$1/.server-info" | head -1 | sed 's/"port"://' + grep -o '"port":[0-9]*' "$1/state/server-info" | head -1 | sed 's/"port"://' } http_check() { @@ -214,11 +216,11 @@ BRAINSTORM_HOST="127.0.0.1" \ BRAINSTORM_URL_HOST="localhost" \ BRAINSTORM_OWNER_PID="" \ BRAINSTORM_PORT=$((49152 + RANDOM % 16383)) \ - node "$SERVER_JS" > "$TEST_DIR/survival/.server.log" 2>&1 & + node "$SERVER_SCRIPT" > "$TEST_DIR/survival/.server.log" 2>&1 & SERVER_PID=$! if ! wait_for_server_info "$TEST_DIR/survival"; then - fail "Server starts successfully" "Server did not write .server-info within 5 seconds" + fail "Server starts successfully" "Server did not write state/server-info within 5 seconds" kill "$SERVER_PID" 2>/dev/null || true SERVER_PID="" else @@ -254,10 +256,15 @@ else SERVER_PID="" fi -# ========== Test 5: Bad OWNER_PID causes shutdown (control) ========== +# ========== Test 5: Dead-at-startup OWNER_PID is logged but does not kill the server ========== +# +# The server validates BRAINSTORM_OWNER_PID at startup. If it's already dead, +# the PID resolution was wrong (common on WSL, Tailscale SSH, cross-user +# scenarios). The server logs 'owner-pid-invalid', disables owner monitoring, +# and continues running. The idle timeout becomes the only shutdown trigger. echo "" -echo "--- Control: Bad OWNER_PID causes shutdown ---" +echo "--- Dead-at-startup OWNER_PID: server survives, logs owner-pid-invalid ---" mkdir -p "$TEST_DIR/control" @@ -272,33 +279,41 @@ BRAINSTORM_HOST="127.0.0.1" \ BRAINSTORM_URL_HOST="localhost" \ BRAINSTORM_OWNER_PID="$BAD_PID" \ BRAINSTORM_PORT=$((49152 + RANDOM % 16383)) \ - node "$SERVER_JS" > "$TEST_DIR/control/.server.log" 2>&1 & + node "$SERVER_SCRIPT" > "$TEST_DIR/control/.server.log" 2>&1 & CONTROL_PID=$! if ! wait_for_server_info "$TEST_DIR/control"; then - fail "Control server starts" "Server did not write .server-info within 5 seconds" + fail "Control server starts" "Server did not write state/server-info within 5 seconds" kill "$CONTROL_PID" 2>/dev/null || true CONTROL_PID="" else - pass "Control server starts with bad OWNER_PID=$BAD_PID" + pass "Control server starts with dead-at-startup OWNER_PID=$BAD_PID" - echo " Waiting ~75s for lifecycle check to kill server..." + echo " Waiting ~75s to verify server survives past lifecycle check..." sleep 75 if kill -0 "$CONTROL_PID" 2>/dev/null; then - fail "Control server self-terminates with bad OWNER_PID" \ - "Server is still alive (expected it to die)" - kill "$CONTROL_PID" 2>/dev/null || true + pass "Server survives with dead-at-startup OWNER_PID (owner monitoring disabled)" else - pass "Control server self-terminates with bad OWNER_PID" + fail "Server survives with dead-at-startup OWNER_PID" \ + "Server died unexpectedly. Log tail: $(tail -5 "$TEST_DIR/control/.server.log" 2>/dev/null)" fi - if grep -q "owner process exited" "$TEST_DIR/control/.server.log" 2>/dev/null; then - pass "Control server logs 'owner process exited'" + if grep -q "owner-pid-invalid" "$TEST_DIR/control/.server.log" 2>/dev/null; then + pass "Server logs 'owner-pid-invalid' for dead-at-startup PID" else - fail "Control server logs 'owner process exited'" \ + fail "Server logs 'owner-pid-invalid' for dead-at-startup PID" \ "Log tail: $(tail -5 "$TEST_DIR/control/.server.log" 2>/dev/null)" fi + + if grep -q "owner process exited" "$TEST_DIR/control/.server.log" 2>/dev/null; then + fail "No spurious 'owner process exited' log" \ + "Found 'owner process exited' but owner monitoring should be disabled" + else + pass "No spurious 'owner process exited' log" + fi + + kill "$CONTROL_PID" 2>/dev/null || true fi wait "$CONTROL_PID" 2>/dev/null || true @@ -309,16 +324,16 @@ CONTROL_PID="" echo "" echo "--- Clean Shutdown ---" -mkdir -p "$TEST_DIR/stop-test" +mkdir -p "$TEST_DIR/stop-test/state" BRAINSTORM_DIR="$TEST_DIR/stop-test" \ BRAINSTORM_HOST="127.0.0.1" \ BRAINSTORM_URL_HOST="localhost" \ BRAINSTORM_OWNER_PID="" \ BRAINSTORM_PORT=$((49152 + RANDOM % 16383)) \ - node "$SERVER_JS" > "$TEST_DIR/stop-test/.server.log" 2>&1 & + node "$SERVER_SCRIPT" > "$TEST_DIR/stop-test/.server.log" 2>&1 & STOP_TEST_PID=$! -echo "$STOP_TEST_PID" > "$TEST_DIR/stop-test/.server.pid" +echo "$STOP_TEST_PID" > "$TEST_DIR/stop-test/state/server.pid" if ! wait_for_server_info "$TEST_DIR/stop-test"; then fail "Stop-test server starts" "Server did not start" diff --git a/tests/claude-code/README.md b/tests/claude-code/README.md index 473f1f28e1..90f1fe1aef 100644 --- a/tests/claude-code/README.md +++ b/tests/claude-code/README.md @@ -115,17 +115,12 @@ Full workflow execution test (~10-30 minutes): - Subagents follow the skill correctly - Final code is functional and tested -#### test-requesting-code-review.sh -Behavioral test for the code reviewer subagent (~5 minutes): -- Builds a tiny project with a baseline commit -- Adds a second commit that plants two real bugs (SQL injection, plaintext password handling) -- Dispatches the code reviewer via the requesting-code-review skill -- Verifies the reviewer flags the planted bugs at Critical/Important severity and refuses to approve - -**What it tests:** -- The skill actually dispatches a working code reviewer subagent -- The reviewer template produces reviewers that catch obvious security bugs -- The reviewer is not sycophantic — it does not approve a diff with planted Critical issues +#### test-worktree-native-preference.sh +RED-GREEN-REFACTOR validation for the using-git-worktrees skill (~5 minutes): +- RED: skill without Step 1a — agent should use `git worktree add` +- GREEN: skill with Step 1a — agent should use the native EnterWorktree tool +- PRESSURE: same as GREEN under urgency framing with pre-existing `.worktrees/` +- Drill scenario `worktree-creation-under-pressure.yaml` covers the PRESSURE phase only ## Adding New Tests diff --git a/tests/claude-code/run-skill-tests.sh b/tests/claude-code/run-skill-tests.sh index 023e979401..cdb8da6e59 100755 --- a/tests/claude-code/run-skill-tests.sh +++ b/tests/claude-code/run-skill-tests.sh @@ -25,7 +25,7 @@ fi # Parse command line arguments VERBOSE=false SPECIFIC_TEST="" -TIMEOUT=300 # Default 5 minute timeout per test +TIMEOUT=600 # Default 10 minute timeout per test RUN_INTEGRATION=false while [[ $# -gt 0 ]]; do @@ -73,13 +73,13 @@ done # List of skill tests to run (fast unit tests) tests=( + "test-worktree-path-policy.sh" "test-subagent-driven-development.sh" ) # Integration tests (slow, full execution) integration_tests=( "test-subagent-driven-development-integration.sh" - "test-requesting-code-review.sh" ) # Add integration tests if requested diff --git a/tests/claude-code/test-document-review-system.sh b/tests/claude-code/test-document-review-system.sh deleted file mode 100755 index ff4c65f36b..0000000000 --- a/tests/claude-code/test-document-review-system.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env bash -# Integration Test: Document Review System -# Actually runs spec/plan review and verifies reviewers catch issues -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -source "$SCRIPT_DIR/test-helpers.sh" - -echo "========================================" -echo " Integration Test: Document Review System" -echo "========================================" -echo "" -echo "This test verifies the document review system by:" -echo " 1. Creating a spec with intentional errors" -echo " 2. Running the spec document reviewer" -echo " 3. Verifying the reviewer catches the errors" -echo "" - -# Create test project -TEST_PROJECT=$(create_test_project) -echo "Test project: $TEST_PROJECT" - -# Trap to cleanup -trap "cleanup_test_project $TEST_PROJECT" EXIT - -cd "$TEST_PROJECT" - -# Create directory structure -mkdir -p docs/superpowers/specs - -# Create a spec document WITH INTENTIONAL ERRORS for the reviewer to catch -cat > docs/superpowers/specs/test-feature-design.md <<'EOF' -# Test Feature Design - -## Overview - -This is a test feature that does something useful. - -## Requirements - -1. The feature should work correctly -2. It should be fast -3. TODO: Add more requirements here - -## Architecture - -The feature will use a simple architecture with: -- A frontend component -- A backend service -- Error handling will be specified later once we understand the failure modes better - -## Data Flow - -Data flows from the frontend to the backend. - -## Testing Strategy - -Tests will be written to cover the main functionality. -EOF - -# Initialize git repo -git init --quiet -git config user.email "test@test.com" -git config user.name "Test User" -git add . -git commit -m "Initial commit with test spec" --quiet - -echo "" -echo "Created test spec with intentional errors:" -echo " - TODO placeholder in Requirements section" -echo " - 'specified later' deferral in Architecture section" -echo "" -echo "Running spec document reviewer..." -echo "" - -# Run Claude to review the spec -OUTPUT_FILE="$TEST_PROJECT/claude-output.txt" - -PROMPT="You are testing the spec document reviewer. - -Read the spec-document-reviewer-prompt.md template in skills/brainstorming/ to understand the review format. - -Then review the spec at $TEST_PROJECT/docs/superpowers/specs/test-feature-design.md using the criteria from that template. - -Look for: -- TODOs, placeholders, 'TBD', incomplete sections -- Sections saying 'to be defined later' or 'will spec when X is done' -- Sections noticeably less detailed than others - -Output your review in the format specified in the template." - -echo "================================================================================" -cd "$SCRIPT_DIR/../.." && timeout 120 claude -p "$PROMPT" --permission-mode bypassPermissions 2>&1 | tee "$OUTPUT_FILE" || { - echo "" - echo "================================================================================" - echo "EXECUTION FAILED (exit code: $?)" - exit 1 -} -echo "================================================================================" - -echo "" -echo "Analyzing reviewer output..." -echo "" - -# Verification tests -FAILED=0 - -echo "=== Verification Tests ===" -echo "" - -# Test 1: Reviewer found the TODO -echo "Test 1: Reviewer found TODO..." -if grep -qi "TODO" "$OUTPUT_FILE" && grep -qi "requirements\|Requirements" "$OUTPUT_FILE"; then - echo " [PASS] Reviewer identified TODO in Requirements section" -else - echo " [FAIL] Reviewer did not identify TODO" - FAILED=$((FAILED + 1)) -fi -echo "" - -# Test 2: Reviewer found the "specified later" deferral -echo "Test 2: Reviewer found 'specified later' deferral..." -if grep -qi "specified later\|later\|defer\|incomplete\|error handling" "$OUTPUT_FILE"; then - echo " [PASS] Reviewer identified deferred content" -else - echo " [FAIL] Reviewer did not identify deferred content" - FAILED=$((FAILED + 1)) -fi -echo "" - -# Test 3: Reviewer output includes Issues section -echo "Test 3: Review output format..." -if grep -qi "issues\|Issues" "$OUTPUT_FILE"; then - echo " [PASS] Review includes Issues section" -else - echo " [FAIL] Review missing Issues section" - FAILED=$((FAILED + 1)) -fi -echo "" - -# Test 4: Reviewer did NOT approve (found issues) -echo "Test 4: Reviewer verdict..." -if grep -qi "Issues Found\|❌\|not approved\|issues found" "$OUTPUT_FILE"; then - echo " [PASS] Reviewer correctly found issues (not approved)" -elif grep -qi "Approved\|✅" "$OUTPUT_FILE" && ! grep -qi "Issues Found\|❌" "$OUTPUT_FILE"; then - echo " [FAIL] Reviewer incorrectly approved spec with errors" - FAILED=$((FAILED + 1)) -else - echo " [PASS] Reviewer identified problems (ambiguous format but found issues)" -fi -echo "" - -# Summary -echo "========================================" -echo " Test Summary" -echo "========================================" -echo "" - -if [ $FAILED -eq 0 ]; then - echo "STATUS: PASSED" - echo "All verification tests passed!" - echo "" - echo "The spec document reviewer correctly:" - echo " ✓ Found TODO placeholder" - echo " ✓ Found 'specified later' deferral" - echo " ✓ Produced properly formatted review" - echo " ✓ Did not approve spec with errors" - exit 0 -else - echo "STATUS: FAILED" - echo "Failed $FAILED verification tests" - echo "" - echo "Output saved to: $OUTPUT_FILE" - echo "" - echo "Review the output to see what went wrong." - exit 1 -fi diff --git a/tests/claude-code/test-helpers.sh b/tests/claude-code/test-helpers.sh index f83a7d222f..1b5ead3b44 100755 --- a/tests/claude-code/test-helpers.sh +++ b/tests/claude-code/test-helpers.sh @@ -9,14 +9,14 @@ run_claude() { local allowed_tools="${3:-}" local output_file=$(mktemp) - # Build command - local cmd="claude -p \"$prompt\"" + # Build command as an argv array so timeout wraps claude directly. + local cmd=(claude -p "$prompt") if [ -n "$allowed_tools" ]; then - cmd="$cmd --allowed-tools=$allowed_tools" + cmd+=(--allowed-tools="$allowed_tools") fi # Run Claude in headless mode with timeout - if timeout "$timeout" bash -c "$cmd" > "$output_file" 2>&1; then + if timeout "$timeout" "${cmd[@]}" > "$output_file" 2>&1; then cat "$output_file" rm -f "$output_file" return 0 diff --git a/tests/claude-code/test-requesting-code-review.sh b/tests/claude-code/test-requesting-code-review.sh deleted file mode 100755 index ca8baafac9..0000000000 --- a/tests/claude-code/test-requesting-code-review.sh +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env bash -# Integration Test: requesting-code-review skill -# Verifies the code reviewer dispatched via the skill catches a planted bug -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -PLUGIN_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" -source "$SCRIPT_DIR/test-helpers.sh" - -echo "========================================" -echo " Integration Test: requesting-code-review" -echo "========================================" -echo "" -echo "This test verifies the code reviewer subagent by:" -echo " 1. Setting up a tiny project with a baseline commit" -echo " 2. Adding a second commit that plants an obvious bug" -echo " 3. Dispatching the code reviewer via the requesting-code-review skill" -echo " 4. Verifying the reviewer flags the planted bug as Critical/Important" -echo "" - -TEST_PROJECT=$(create_test_project) -echo "Test project: $TEST_PROJECT" -trap "cleanup_test_project $TEST_PROJECT" EXIT - -cd "$TEST_PROJECT" - -# Baseline: a small "safe" implementation -mkdir -p src -cat > src/db.js <<'EOF' -import { Database } from "./database-driver.js"; - -const db = new Database(); - -export async function findUserByEmail(email) { - if (typeof email !== "string" || !email) { - throw new Error("email required"); - } - return db.query( - "SELECT id, email, created_at FROM users WHERE email = ?", - [email], - ); -} -EOF - -cat > package.json <<'EOF' -{ "name": "test-codereview", "version": "1.0.0", "type": "module" } -EOF - -git init --quiet -git config user.email "test@test.com" -git config user.name "Test User" -git add . -git commit -m "Initial: parameterized findUserByEmail" --quiet -BASE_SHA=$(git rev-parse HEAD) - -# Second commit: plant two real bugs -# 1. SQL injection — switch from parameterized to string concatenation -# 2. Logs the user's password hash on every successful login -cat > src/db.js <<'EOF' -import { Database } from "./database-driver.js"; - -const db = new Database(); - -export async function findUserByEmail(email) { - return db.query( - "SELECT id, email, password_hash, created_at FROM users WHERE email = '" + email + "'", - ); -} - -export async function login(email, password) { - const user = await findUserByEmail(email); - if (user && user.password_hash === hash(password)) { - console.log("login success", { email, password_hash: user.password_hash }); - return user; - } - return null; -} - -function hash(s) { return s; } -EOF - -git add . -git commit -m "Refactor user lookup, add login" --quiet -HEAD_SHA=$(git rev-parse HEAD) - -echo "" -echo "Planted bugs in $BASE_SHA..$HEAD_SHA:" -echo " - SQL injection (string concat instead of parameterized query)" -echo " - Password hash logged in plaintext on every successful login" -echo " - hash() is the identity function (passwords stored & compared in plaintext)" -echo "" - -OUTPUT_FILE="$TEST_PROJECT/claude-output.txt" - -PROMPT="I just finished a refactor. The change is between commits $BASE_SHA and $HEAD_SHA on the current branch. - -Use the superpowers:requesting-code-review skill to review these changes before I merge. Follow the skill exactly: dispatch the code reviewer subagent with the template, give the subagent the SHA range, and report back what it found. - -Print the reviewer's full output." - -# Run claude from inside the test project so its session JSONL lands in a -# project-specific directory under ~/.claude/projects/, isolated from any -# other concurrent claude sessions. -echo "Running Claude (plugin-dir: $PLUGIN_DIR, cwd: $TEST_PROJECT)..." -echo "================================================================================" -cd "$TEST_PROJECT" && timeout 600 claude -p "$PROMPT" \ - --plugin-dir "$PLUGIN_DIR" \ - --permission-mode bypassPermissions 2>&1 | tee "$OUTPUT_FILE" || { - echo "" - echo "================================================================================" - echo "EXECUTION FAILED (exit code: $?)" - exit 1 -} -echo "================================================================================" - -echo "" -echo "Analyzing reviewer output..." -echo "" - -# Find the session transcript. Because we ran claude from $TEST_PROJECT (a -# unique tmp dir), its sessions live in their own ~/.claude/projects/ folder. -# Resolve the real path (macOS mktemp returns /var/... but claude normalizes -# it to /private/var/...) and replicate claude's normalization (every -# non-alphanumeric char becomes `-`). -TEST_PROJECT_REAL=$(cd "$TEST_PROJECT" && pwd -P) -SESSION_DIR="$HOME/.claude/projects/$(echo "$TEST_PROJECT_REAL" | sed 's|[^a-zA-Z0-9]|-|g')" -# `|| true` prevents pipefail killing the script if ls gets SIGPIPE'd by head. -SESSION_FILE=$(ls -t "$SESSION_DIR"/*.jsonl 2>/dev/null | head -1 || true) - -FAILED=0 - -echo "=== Verification Tests ===" -echo "" - -# Test 1: Skill was actually invoked, and a subagent was actually dispatched -echo "Test 1: requesting-code-review skill invoked + reviewer subagent dispatched..." -if [ -z "$SESSION_FILE" ] || [ ! -f "$SESSION_FILE" ]; then - echo " [FAIL] Could not locate session transcript in $SESSION_DIR" - FAILED=$((FAILED + 1)) -elif ! grep -q '"skill":"superpowers:requesting-code-review"' "$SESSION_FILE"; then - echo " [FAIL] requesting-code-review skill was not invoked" - echo " Session: $SESSION_FILE" - FAILED=$((FAILED + 1)) -elif ! grep -q '"name":"Agent"' "$SESSION_FILE"; then - echo " [FAIL] Skill ran but no subagent was dispatched" - FAILED=$((FAILED + 1)) -else - echo " [PASS] Skill invoked and subagent dispatched" -fi -echo "" - -# Test 2: Reviewer caught the SQL injection -echo "Test 2: SQL injection flagged..." -if grep -qiE "sql injection|injection|string concat|parameterize|prepared statement|sanitiz" "$OUTPUT_FILE"; then - echo " [PASS] Reviewer flagged the SQL injection vector" -else - echo " [FAIL] Reviewer missed the SQL injection — most obvious planted bug" - FAILED=$((FAILED + 1)) -fi -echo "" - -# Test 3: Reviewer caught the credential / password issue (either logging or no real hashing) -echo "Test 3: Credential handling issue flagged..." -if grep -qiE "password|credential|secret|plaintext|log.*hash|hash.*log|sensitive" "$OUTPUT_FILE"; then - echo " [PASS] Reviewer flagged a credential / password handling issue" -else - echo " [FAIL] Reviewer missed the password/credential issues" - FAILED=$((FAILED + 1)) -fi -echo "" - -# Test 4: Reviewer marked at least one issue as Critical or Important (not just Minor) -echo "Test 4: Severity classification..." -if grep -qiE "critical|important|severe|high.*risk|security" "$OUTPUT_FILE"; then - echo " [PASS] Reviewer classified findings at Critical/Important severity" -else - echo " [FAIL] Reviewer did not classify findings as Critical or Important" - FAILED=$((FAILED + 1)) -fi -echo "" - -# Test 5: Reviewer did NOT approve the diff for merge -echo "Test 5: Reviewer verdict..." -# A correct reviewer says No or "With fixes". A broken/sycophantic reviewer says Yes/Ready. -if grep -qiE "ready to merge.*yes|approved.*for merge|^\s*yes\s*$|safe to merge" "$OUTPUT_FILE" \ - && ! grep -qiE "ready to merge.*no|with fixes|do not merge|not ready|block.*merge" "$OUTPUT_FILE"; then - echo " [FAIL] Reviewer approved a diff with planted Critical bugs" - FAILED=$((FAILED + 1)) -else - echo " [PASS] Reviewer did not approve the diff" -fi -echo "" - -echo "========================================" -echo " Test Summary" -echo "========================================" -echo "" - -if [ $FAILED -eq 0 ]; then - echo "STATUS: PASSED" - echo "The code reviewer correctly:" - echo " ✓ Was dispatched via the requesting-code-review skill" - echo " ✓ Flagged the SQL injection" - echo " ✓ Flagged the credential handling issues" - echo " ✓ Classified findings at Critical/Important severity" - echo " ✓ Did not approve the diff for merge" - exit 0 -else - echo "STATUS: FAILED" - echo "Failed $FAILED verification tests" - echo "" - echo "Output saved to: $OUTPUT_FILE" - exit 1 -fi diff --git a/tests/claude-code/test-subagent-driven-development-integration.sh b/tests/claude-code/test-subagent-driven-development-integration.sh index 95a551bca8..a783648314 100755 --- a/tests/claude-code/test-subagent-driven-development-integration.sh +++ b/tests/claude-code/test-subagent-driven-development-integration.sh @@ -1,6 +1,17 @@ #!/usr/bin/env bash # Integration Test: subagent-driven-development workflow # Actually executes a plan and verifies the new workflow behaviors +# +# Drill coverage: evals/scenarios/sdd-rejects-extra-features.yaml covers the +# YAGNI enforcement subset (forbidden exports + reviewer-as-gate semantics) +# and is stricter on that axis. This bash test additionally asserts: +# - >=3 git commits (initial + per-task commits, exercising SDD's +# commit-per-task workflow shape) +# - >=2 Claude Code subagent dispatches via Agent or Task (drill only asserts >=1) +# - Claude Code task-tracking tool usage (drill makes no assertion) +# - test/math.test.js exists (drill relies on `npm test` succeeding) +# - analyze-token-usage.py token-budget telemetry +# Kept until those assertions are added to drill or explicitly retired. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -213,13 +224,13 @@ else fi echo "" -# Test 3: TodoWrite was used for tracking +# Test 3: Claude Code task-tracking tool was used echo "Test 3: Task tracking..." -todo_count=$(grep -c '"name":"TodoWrite"' "$SESSION_FILE" || echo "0") +todo_count=$(grep -cE '"name":"(TodoWrite|TaskCreate|TaskUpdate|TaskList|TaskGet)"' "$SESSION_FILE" || echo "0") if [ "$todo_count" -ge 1 ]; then - echo " [PASS] TodoWrite used $todo_count time(s) for task tracking" + echo " [PASS] Task tracking used $todo_count time(s)" else - echo " [FAIL] TodoWrite not used" + echo " [FAIL] No Claude Code task-tracking tool used" FAILED=$((FAILED + 1)) fi echo "" diff --git a/tests/claude-code/test-subagent-driven-development.sh b/tests/claude-code/test-subagent-driven-development.sh index 20d8d4c7e6..d8f3e10ce6 100755 --- a/tests/claude-code/test-subagent-driven-development.sh +++ b/tests/claude-code/test-subagent-driven-development.sh @@ -1,18 +1,26 @@ #!/usr/bin/env bash # Test: subagent-driven-development skill # Verifies that the skill is loaded and follows correct workflow +# +# No drill coverage: this test asks the agent to *describe* SDD (string- +# matches its verbal explanation against expected keywords like +# "self-review", "skeptical", "worktree", "Step 1", "loop"). Drill scenarios +# test behavior (real subagent dispatch, plan-following, review loops), +# not description-recall. Kept by design. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "$SCRIPT_DIR/test-helpers.sh" +CLAUDE_PROMPT_TIMEOUT="${CLAUDE_PROMPT_TIMEOUT:-90}" + echo "=== Test: subagent-driven-development skill ===" echo "" # Test 1: Verify skill can be loaded echo "Test 1: Skill loading..." -output=$(run_claude "What is the subagent-driven-development skill? Describe its key steps briefly." 30) +output=$(run_claude "What is the subagent-driven-development skill? Describe its key steps briefly." "$CLAUDE_PROMPT_TIMEOUT") if assert_contains "$output" "subagent-driven-development\|Subagent-Driven Development\|Subagent Driven" "Skill is recognized"; then : # pass @@ -31,9 +39,11 @@ echo "" # Test 2: Verify skill describes correct workflow order echo "Test 2: Workflow ordering..." -output=$(run_claude "In the subagent-driven-development skill, what comes first: spec compliance review or code quality review? Be specific about the order." 30) +output=$(run_claude "In the subagent-driven-development skill, what comes first: spec compliance review or code quality review? Answer using exactly this structure: +First: +Second: " "$CLAUDE_PROMPT_TIMEOUT") -if assert_order "$output" "spec.*compliance" "code.*quality" "Spec compliance before code quality"; then +if assert_order "$output" "First:.*spec.*compliance" "Second:.*code.*quality" "Spec compliance before code quality"; then : # pass else exit 1 @@ -44,15 +54,17 @@ echo "" # Test 3: Verify self-review is mentioned echo "Test 3: Self-review requirement..." -output=$(run_claude "Does the subagent-driven-development skill require implementers to do self-review? What should they check?" 30) +output=$(run_claude "Does the subagent-driven-development skill require implementers to self-review before handoff, and can self-review replace the external reviews? Answer using exactly this structure: +Self-review required: +Self-review replaces external review: " "$CLAUDE_PROMPT_TIMEOUT") -if assert_contains "$output" "self-review\|self review" "Mentions self-review"; then +if assert_contains "$output" "Self-review required:.*yes" "Mentions self-review"; then : # pass else exit 1 fi -if assert_contains "$output" "completeness\|Completeness" "Checks completeness"; then +if assert_contains "$output" "Self-review replaces external review:.*no" "Self-review does not replace external review"; then : # pass else exit 1 @@ -63,7 +75,7 @@ echo "" # Test 4: Verify plan is read once echo "Test 4: Plan reading efficiency..." -output=$(run_claude "In subagent-driven-development, how many times should the controller read the plan file? When does this happen?" 30) +output=$(run_claude "In subagent-driven-development, how many times should the controller read the plan file? When does this happen?" "$CLAUDE_PROMPT_TIMEOUT") if assert_contains "$output" "once\|one time\|single" "Read plan once"; then : # pass @@ -82,7 +94,7 @@ echo "" # Test 5: Verify spec compliance reviewer is skeptical echo "Test 5: Spec compliance reviewer mindset..." -output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" 30) +output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" "$CLAUDE_PROMPT_TIMEOUT") if assert_contains "$output" "not trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then : # pass @@ -101,7 +113,7 @@ echo "" # Test 6: Verify review loops echo "Test 6: Review loop requirements..." -output=$(run_claude "In subagent-driven-development, what happens if a reviewer finds issues? Is it a one-time review or a loop?" 30) +output=$(run_claude "In subagent-driven-development, what happens if a reviewer finds issues? Is it a one-time review or a loop?" "$CLAUDE_PROMPT_TIMEOUT") if assert_contains "$output" "loop\|again\|repeat\|until.*approved\|until.*compliant" "Review loops mentioned"; then : # pass @@ -120,7 +132,9 @@ echo "" # Test 7: Verify full task text is provided echo "Test 7: Task context provision..." -output=$(run_claude "In subagent-driven-development, how does the controller provide task information to the implementer subagent? Does it make them read a file or provide it directly?" 30) +output=$(run_claude "In subagent-driven-development, how does the controller provide task information to the implementer subagent? Answer using exactly this structure: +Controller provides: +Implementer must read plan file: " "$CLAUDE_PROMPT_TIMEOUT") if assert_contains "$output" "provide.*directly\|full.*text\|paste\|include.*prompt" "Provides text directly"; then : # pass @@ -128,7 +142,7 @@ else exit 1 fi -if assert_not_contains "$output" "read.*file\|open.*file" "Doesn't make subagent read file"; then +if assert_contains "$output" "Implementer must read plan file:.*no" "Doesn't make subagent read file"; then : # pass else exit 1 @@ -139,7 +153,7 @@ echo "" # Test 8: Verify worktree requirement echo "Test 8: Worktree requirement..." -output=$(run_claude "What workflow skills are required before using subagent-driven-development? List any prerequisites or required skills." 30) +output=$(run_claude "What workflow skills are required before using subagent-driven-development? List any prerequisites or required skills." "$CLAUDE_PROMPT_TIMEOUT") if assert_contains "$output" "using-git-worktrees\|worktree" "Mentions worktree requirement"; then : # pass @@ -152,7 +166,7 @@ echo "" # Test 9: Verify main branch warning echo "Test 9: Main branch red flag..." -output=$(run_claude "In subagent-driven-development, is it okay to start implementation directly on the main branch?" 30) +output=$(run_claude "In subagent-driven-development, is it okay to start implementation directly on the main branch?" "$CLAUDE_PROMPT_TIMEOUT") if assert_contains "$output" "worktree\|feature.*branch\|not.*main\|never.*main\|avoid.*main\|don't.*main\|consent\|permission" "Warns against main branch"; then : # pass diff --git a/tests/claude-code/test-worktree-native-preference.sh b/tests/claude-code/test-worktree-native-preference.sh index cbfe7f293e..077ea19c36 100755 --- a/tests/claude-code/test-worktree-native-preference.sh +++ b/tests/claude-code/test-worktree-native-preference.sh @@ -2,6 +2,11 @@ # Test: Does the agent prefer native worktree tools (EnterWorktree) over git worktree add? # Framework: RED-GREEN-REFACTOR per testing-skills-with-subagents.md # +# Drill coverage: evals/scenarios/worktree-creation-under-pressure.yaml lifts +# only the PRESSURE phase (existing .worktrees/ + urgency framing). The RED +# and GREEN baselines below are not covered by drill — kept here so the +# RED-GREEN-REFACTOR validation remains rerunnable end-to-end. +# # RED: Skill without Step 1a (no native tool preference). Agent should use git worktree add. # GREEN: Skill with Step 1a (explicit tool naming + consent bridge). Agent should use EnterWorktree. # PRESSURE: Same as GREEN but under time pressure with existing .worktrees/ dir. diff --git a/tests/claude-code/test-worktree-path-policy.sh b/tests/claude-code/test-worktree-path-policy.sh new file mode 100755 index 0000000000..58caad7a09 --- /dev/null +++ b/tests/claude-code/test-worktree-path-policy.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Regression check: Superpowers should not route new worktrees through the old +# global worktree directory. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +USING_SKILL="$REPO_ROOT/skills/using-git-worktrees/SKILL.md" +FINISHING_SKILL="$REPO_ROOT/skills/finishing-a-development-branch/SKILL.md" +ROTOTILL_SPEC="$REPO_ROOT/docs/superpowers/specs/2026-04-06-worktree-rototill-design.md" +ROTOTILL_PLAN="$REPO_ROOT/docs/superpowers/plans/2026-04-06-worktree-rototill.md" + +failures=0 + +assert_contains() { + local file="$1" + local pattern="$2" + local label="$3" + + if grep -Fq "$pattern" "$file"; then + echo " [PASS] $label" + else + echo " [FAIL] $label" + echo " Expected to find: $pattern" + echo " In file: $file" + failures=$((failures + 1)) + fi +} + +assert_not_contains() { + local file="$1" + local pattern="$2" + local label="$3" + + if grep -Fq "$pattern" "$file"; then + echo " [FAIL] $label" + echo " Did not expect to find: $pattern" + echo " In file: $file" + failures=$((failures + 1)) + else + echo " [PASS] $label" + fi +} + +echo "=== Worktree Path Policy Test ===" +echo "" + +assert_not_contains "$USING_SKILL" "~/.config/superpowers/worktrees" "using-git-worktrees does not mention old global path" +assert_not_contains "$USING_SKILL" "global legacy" "using-git-worktrees does not use unclear global legacy shorthand" +assert_not_contains "$USING_SKILL" "Global path" "using-git-worktrees has no global path quick-reference row" +assert_contains "$USING_SKILL" 'default to `.worktrees/` at the project root' "using-git-worktrees defaults new manual worktrees to .worktrees/" + +assert_not_contains "$FINISHING_SKILL" "~/.config/superpowers/worktrees" "finishing-a-development-branch does not treat old global path as owned" +assert_contains "$FINISHING_SKILL" '`.worktrees/` or `worktrees/`' "finishing-a-development-branch keeps project-local cleanup ownership" + +assert_not_contains "$ROTOTILL_SPEC" "~/.config/superpowers/worktrees" "rototill spec does not preserve old global path policy" +assert_not_contains "$ROTOTILL_PLAN" "~/.config/superpowers/worktrees" "rototill plan does not preserve old global path policy" +assert_not_contains "$ROTOTILL_PLAN" "legacy path compat" "rototill plan does not advertise legacy path compatibility" + +echo "" + +if [ "$failures" -gt 0 ]; then + echo "STATUS: FAILED ($failures failures)" + exit 1 +fi + +echo "STATUS: PASSED" diff --git a/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh b/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh index 441230e144..83c552fdee 100755 --- a/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh +++ b/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh @@ -177,7 +177,10 @@ write_upstream_fixture() { "$repo/.codex-plugin" \ "$repo/.private-journal" \ "$repo/assets" \ + "$repo/evals/drill" \ + "$repo/hooks" \ "$repo/scripts" \ + "$repo/skills/brainstorming/scripts/vendor" \ "$repo/skills/example" if [[ "$with_pure_ignored" == "1" ]]; then @@ -215,11 +218,70 @@ EOF EOF printf 'png fixture\n' > "$repo/assets/app-icon.png" + printf 'eval harness fixture\n' > "$repo/evals/drill/README.md" + + cat > "$repo/hooks/hooks-codex.json" <<'EOF' +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume|clear", + "hooks": [ + { + "type": "command", + "command": "\"${PLUGIN_ROOT}/hooks/run-hook.cmd\" session-start-codex", + "async": false + } + ] + } + ] + } +} +EOF + + cat > "$repo/hooks/session-start" <<'EOF' +#!/usr/bin/env sh +echo "session-start fixture" +EOF + cat > "$repo/hooks/session-start-codex" <<'EOF' +#!/usr/bin/env sh +echo "session-start-codex fixture" +EOF + + cat > "$repo/hooks/run-hook.cmd" <<'EOF' +@echo off +echo run-hook fixture +EOF + chmod +x "$repo/hooks/session-start" "$repo/hooks/session-start-codex" "$repo/hooks/run-hook.cmd" cat > "$repo/skills/example/SKILL.md" <<'EOF' # Example Skill Fixture content. +EOF + + cat > "$repo/skills/brainstorming/scripts/server.cjs" <<'EOF' +console.log('fixture server') +EOF + + cat > "$repo/skills/brainstorming/scripts/helper.js" <<'EOF' +window.fixtureHelper = true +EOF + + cat > "$repo/skills/brainstorming/scripts/frame-template.html" <<'EOF' + +EOF + + printf 'fixture alpine\n' > "$repo/skills/brainstorming/scripts/vendor/alpine.js" + + cat > "$repo/skills/brainstorming/scripts/vendor/alpine.provenance.json" <<'EOF' +{"name":"alpinejs","version":"3.15.12","localPath":"skills/brainstorming/scripts/vendor/alpine.js","sha256":"fixture","approvalArtifact":"SUP-215"} +EOF + + cat > "$repo/skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md" <<'EOF' +# Third-Party Notices + +Alpine.js fixture notice. EOF printf 'tracked keep\n' > "$repo/.private-journal/keep.txt" @@ -233,8 +295,19 @@ EOF .gitignore \ assets/app-icon.png \ assets/superpowers-small.svg \ + evals/drill/README.md \ + hooks/hooks-codex.json \ + hooks/run-hook.cmd \ + hooks/session-start \ + hooks/session-start-codex \ package.json \ scripts/sync-to-codex-plugin.sh \ + skills/brainstorming/scripts/server.cjs \ + skills/brainstorming/scripts/helper.js \ + skills/brainstorming/scripts/frame-template.html \ + skills/brainstorming/scripts/vendor/alpine.js \ + skills/brainstorming/scripts/vendor/alpine.provenance.json \ + skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md \ skills/example/SKILL.md git -C "$repo" add -f .private-journal/keep.txt @@ -290,6 +363,8 @@ write_synced_destination_fixture() { "$repo/plugins/superpowers/.codex-plugin" \ "$repo/plugins/superpowers/.private-journal" \ "$repo/plugins/superpowers/assets" \ + "$repo/plugins/superpowers/hooks" \ + "$repo/plugins/superpowers/skills/brainstorming/scripts/vendor" \ "$repo/plugins/superpowers/skills/example/agents" \ "$repo/plugins/superpowers/skills/example" @@ -306,10 +381,68 @@ EOF printf 'png fixture\n' > "$repo/plugins/superpowers/assets/app-icon.png" + cat > "$repo/plugins/superpowers/hooks/hooks-codex.json" <<'EOF' +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume|clear", + "hooks": [ + { + "type": "command", + "command": "\"${PLUGIN_ROOT}/hooks/run-hook.cmd\" session-start-codex", + "async": false + } + ] + } + ] + } +} +EOF + + cat > "$repo/plugins/superpowers/hooks/session-start" <<'EOF' +#!/usr/bin/env sh +echo "session-start fixture" +EOF + cat > "$repo/plugins/superpowers/hooks/session-start-codex" <<'EOF' +#!/usr/bin/env sh +echo "session-start-codex fixture" +EOF + + cat > "$repo/plugins/superpowers/hooks/run-hook.cmd" <<'EOF' +@echo off +echo run-hook fixture +EOF + chmod +x "$repo/plugins/superpowers/hooks/session-start" "$repo/plugins/superpowers/hooks/session-start-codex" "$repo/plugins/superpowers/hooks/run-hook.cmd" + cat > "$repo/plugins/superpowers/skills/example/SKILL.md" <<'EOF' # Example Skill Fixture content. +EOF + + cat > "$repo/plugins/superpowers/skills/brainstorming/scripts/server.cjs" <<'EOF' +console.log('fixture server') +EOF + + cat > "$repo/plugins/superpowers/skills/brainstorming/scripts/helper.js" <<'EOF' +window.fixtureHelper = true +EOF + + cat > "$repo/plugins/superpowers/skills/brainstorming/scripts/frame-template.html" <<'EOF' + +EOF + + printf 'fixture alpine\n' > "$repo/plugins/superpowers/skills/brainstorming/scripts/vendor/alpine.js" + + cat > "$repo/plugins/superpowers/skills/brainstorming/scripts/vendor/alpine.provenance.json" <<'EOF' +{"name":"alpinejs","version":"3.15.12","localPath":"skills/brainstorming/scripts/vendor/alpine.js","sha256":"fixture","approvalArtifact":"SUP-215"} +EOF + + cat > "$repo/plugins/superpowers/skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md" <<'EOF' +# Third-Party Notices + +Alpine.js fixture notice. EOF cat > "$repo/plugins/superpowers/skills/example/agents/openai.yaml" <<'EOF' @@ -324,6 +457,16 @@ EOF plugins/superpowers/.codex-plugin/plugin.json \ plugins/superpowers/assets/app-icon.png \ plugins/superpowers/assets/superpowers-small.svg \ + plugins/superpowers/hooks/hooks-codex.json \ + plugins/superpowers/hooks/run-hook.cmd \ + plugins/superpowers/hooks/session-start \ + plugins/superpowers/hooks/session-start-codex \ + plugins/superpowers/skills/brainstorming/scripts/server.cjs \ + plugins/superpowers/skills/brainstorming/scripts/helper.js \ + plugins/superpowers/skills/brainstorming/scripts/frame-template.html \ + plugins/superpowers/skills/brainstorming/scripts/vendor/alpine.js \ + plugins/superpowers/skills/brainstorming/scripts/vendor/alpine.provenance.json \ + plugins/superpowers/skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md \ plugins/superpowers/skills/example/agents/openai.yaml \ plugins/superpowers/skills/example/SKILL.md \ plugins/superpowers/.private-journal/keep.txt @@ -342,6 +485,46 @@ write_stale_ignored_destination_fixture() { commit_fixture "$repo" "Initial stale ignored destination fixture" } +write_outdated_destination_fixture() { + local repo="$1" + + mkdir -p \ + "$repo/plugins/superpowers/.codex-plugin" \ + "$repo/plugins/superpowers/assets" \ + "$repo/plugins/superpowers/skills/example" + + cat > "$repo/plugins/superpowers/.codex-plugin/plugin.json" <<'EOF' +{ + "name": "superpowers", + "version": "0.0.1" +} +EOF + + printf 'old png fixture\n' > "$repo/plugins/superpowers/assets/app-icon.png" + + cat > "$repo/plugins/superpowers/skills/example/SKILL.md" <<'EOF' +# Example Skill + +Old destination content. +EOF + + git -C "$repo" add \ + plugins/superpowers/.codex-plugin/plugin.json \ + plugins/superpowers/assets/app-icon.png \ + plugins/superpowers/skills/example/SKILL.md + + commit_fixture "$repo" "Initial outdated destination fixture" +} + +attach_origin_remote() { + local repo="$1" + local remote="$2" + + git init -q --bare "$remote" + git -C "$repo" remote add origin "$remote" + git -C "$repo" push -u origin main --quiet +} + write_fake_gh() { local bin_dir="$1" @@ -355,6 +538,29 @@ if [[ "${1:-}" == "auth" && "${2:-}" == "status" ]]; then exit 0 fi +if [[ "${1:-}" == "pr" && "${2:-}" == "create" ]]; then + shift 2 + body="" + while [[ $# -gt 0 ]]; do + case "$1" in + --body) + body="${2:-}" + shift 2 + ;; + *) + shift + ;; + esac + done + + if [[ -n "${FAKE_GH_PR_BODY_FILE:-}" ]]; then + printf '%s' "$body" > "$FAKE_GH_PR_BODY_FILE" + fi + + echo "https://github.com/prime-radiant-inc/openai-codex-plugins/pull/123" + exit 0 +fi + echo "unexpected gh invocation: $*" >&2 exit 1 EOF @@ -403,6 +609,24 @@ run_apply() { PATH="$fake_bin:$PATH" "$BASH_UNDER_TEST" "$upstream/scripts/sync-to-codex-plugin.sh" -y --local "$dest" 2>&1 } +run_apply_with_pr_capture() { + local upstream="$1" + local dest="$2" + local fake_bin="$3" + local body_file="$4" + + FAKE_GH_PR_BODY_FILE="$body_file" PATH="$fake_bin:$PATH" "$BASH_UNDER_TEST" "$upstream/scripts/sync-to-codex-plugin.sh" -y --local "$dest" 2>&1 +} + +run_bootstrap_apply_with_pr_capture() { + local upstream="$1" + local dest="$2" + local fake_bin="$3" + local body_file="$4" + + FAKE_GH_PR_BODY_FILE="$body_file" PATH="$fake_bin:$PATH" "$BASH_UNDER_TEST" "$upstream/scripts/sync-to-codex-plugin.sh" -y --bootstrap --local "$dest" 2>&1 +} + run_help() { local upstream="$1" local fake_bin="$2" @@ -428,11 +652,15 @@ main() { local stale_dest local dirty_apply_dest local dirty_apply_dest_branch + local changed_apply_dest + local changed_apply_remote local noop_apply_dest local noop_apply_dest_branch local fake_bin local bootstrap_dest local bootstrap_dest_branch + local bootstrap_apply_dest + local bootstrap_apply_remote local preview_status local preview_output local preview_section @@ -447,12 +675,26 @@ main() { local stale_preview_section local dirty_apply_status local dirty_apply_output + local changed_apply_status + local changed_apply_output + local changed_apply_pr_body_path + local changed_apply_pr_body + local bootstrap_apply_status + local bootstrap_apply_output + local bootstrap_apply_pr_body_path + local bootstrap_apply_pr_body local noop_apply_status local noop_apply_output local help_output local script_source local dirty_skill_path + local changed_apply_alpine_path + local changed_apply_alpine_provenance_path + local changed_apply_alpine_notice_path local noop_openai_metadata_path + local noop_alpine_path + local noop_alpine_provenance_path + local noop_alpine_notice_path echo "=== Test: sync-to-codex-plugin dry-run regression ===" @@ -466,9 +708,13 @@ main() { stale_dest="$TEST_ROOT/stale-destination" dirty_apply_dest="$TEST_ROOT/dirty-apply-destination" dirty_apply_dest_branch="fixture/dirty-apply-target" + changed_apply_dest="$TEST_ROOT/changed-apply-destination" + changed_apply_remote="$TEST_ROOT/changed-apply-remote.git" noop_apply_dest="$TEST_ROOT/noop-apply-destination" noop_apply_dest_branch="fixture/noop-apply-target" bootstrap_dest="$TEST_ROOT/bootstrap-destination" + bootstrap_apply_dest="$TEST_ROOT/bootstrap-apply-destination" + bootstrap_apply_remote="$TEST_ROOT/bootstrap-apply-remote.git" dest_branch="fixture/preview-target" bootstrap_dest_branch="fixture/bootstrap-preview-target" fake_bin="$TEST_ROOT/bin" @@ -496,6 +742,10 @@ main() { checkout_fixture_branch "$dirty_apply_dest" "$dirty_apply_dest_branch" dirty_tracked_destination_skill "$dirty_apply_dest" + init_repo "$changed_apply_dest" + write_outdated_destination_fixture "$changed_apply_dest" + attach_origin_remote "$changed_apply_dest" "$changed_apply_remote" + init_repo "$noop_apply_dest" write_synced_destination_fixture "$noop_apply_dest" checkout_fixture_branch "$noop_apply_dest" "$noop_apply_dest_branch" @@ -504,6 +754,10 @@ main() { write_bootstrap_destination_fixture "$bootstrap_dest" checkout_fixture_branch "$bootstrap_dest" "$bootstrap_dest_branch" + init_repo "$bootstrap_apply_dest" + write_bootstrap_destination_fixture "$bootstrap_apply_dest" + attach_origin_remote "$bootstrap_apply_dest" "$bootstrap_apply_remote" + write_fake_gh "$fake_bin" # This regression test is about dry-run content, so capture the preview @@ -519,6 +773,12 @@ main() { stale_preview_status=$? dirty_apply_output="$(run_apply "$upstream" "$dirty_apply_dest" "$fake_bin")" dirty_apply_status=$? + changed_apply_pr_body_path="$TEST_ROOT/changed-apply-pr-body.md" + changed_apply_output="$(run_apply_with_pr_capture "$upstream" "$changed_apply_dest" "$fake_bin" "$changed_apply_pr_body_path")" + changed_apply_status=$? + bootstrap_apply_pr_body_path="$TEST_ROOT/bootstrap-apply-pr-body.md" + bootstrap_apply_output="$(run_bootstrap_apply_with_pr_capture "$upstream" "$bootstrap_apply_dest" "$fake_bin" "$bootstrap_apply_pr_body_path")" + bootstrap_apply_status=$? noop_apply_output="$(run_apply "$upstream" "$noop_apply_dest" "$fake_bin")" noop_apply_status=$? missing_manifest_output="$(run_preview_without_manifest "$upstream" "$dest" "$fake_bin")" @@ -529,7 +789,15 @@ main() { preview_section="$(printf '%s\n' "$preview_output" | sed -n '/^=== Preview (rsync --dry-run) ===$/,/^=== End preview ===$/p')" stale_preview_section="$(printf '%s\n' "$stale_preview_output" | sed -n '/^=== Preview (rsync --dry-run) ===$/,/^=== End preview ===$/p')" dirty_skill_path="$dirty_apply_dest/plugins/superpowers/skills/example/SKILL.md" + changed_apply_alpine_path="$changed_apply_dest/plugins/superpowers/skills/brainstorming/scripts/vendor/alpine.js" + changed_apply_alpine_provenance_path="$changed_apply_dest/plugins/superpowers/skills/brainstorming/scripts/vendor/alpine.provenance.json" + changed_apply_alpine_notice_path="$changed_apply_dest/plugins/superpowers/skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md" + changed_apply_pr_body="$(cat "$changed_apply_pr_body_path" 2>/dev/null || true)" + bootstrap_apply_pr_body="$(cat "$bootstrap_apply_pr_body_path" 2>/dev/null || true)" noop_openai_metadata_path="$noop_apply_dest/plugins/superpowers/skills/example/agents/openai.yaml" + noop_alpine_path="$noop_apply_dest/plugins/superpowers/skills/brainstorming/scripts/vendor/alpine.js" + noop_alpine_provenance_path="$noop_apply_dest/plugins/superpowers/skills/brainstorming/scripts/vendor/alpine.provenance.json" + noop_alpine_notice_path="$noop_apply_dest/plugins/superpowers/skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md" echo "" echo "Preview assertions..." @@ -539,12 +807,23 @@ main() { assert_contains "$preview_section" ".codex-plugin/plugin.json" "Preview includes manifest path" assert_contains "$preview_section" "assets/superpowers-small.svg" "Preview includes SVG asset" assert_contains "$preview_section" "assets/app-icon.png" "Preview includes PNG asset" + assert_contains "$preview_section" "hooks/hooks-codex.json" "Preview includes Codex hook manifest" + assert_contains "$preview_section" "hooks/session-start" "Preview includes session-start hook" + assert_contains "$preview_section" "hooks/session-start-codex" "Preview includes Codex session-start hook" + assert_contains "$preview_section" "hooks/run-hook.cmd" "Preview includes hook command wrapper" assert_contains "$preview_section" ".private-journal/keep.txt" "Preview includes tracked ignored file" assert_not_contains "$preview_section" ".private-journal/leak.txt" "Preview excludes ignored untracked file" assert_not_contains "$preview_section" "ignored-cache/" "Preview excludes pure ignored directories" + assert_not_contains "$preview_section" "evals/" "Preview excludes eval harness" assert_not_contains "$preview_output" "Overlay file (.codex-plugin/plugin.json) will be regenerated" "Preview omits overlay regeneration note" assert_not_contains "$preview_output" "Assets (superpowers-small.svg, app-icon.png) will be seeded from" "Preview omits assets seeding note" assert_contains "$preview_section" "skills/example/SKILL.md" "Preview reflects dirty tracked destination file" + assert_contains "$preview_section" "skills/brainstorming/scripts/server.cjs" "Preview includes skill-local server runtime" + assert_contains "$preview_section" "skills/brainstorming/scripts/helper.js" "Preview includes skill-local helper runtime" + assert_contains "$preview_section" "skills/brainstorming/scripts/frame-template.html" "Preview includes skill-local frame template" + assert_contains "$preview_section" "skills/brainstorming/scripts/vendor/alpine.js" "Preview includes vendored Alpine" + assert_contains "$preview_section" "skills/brainstorming/scripts/vendor/alpine.provenance.json" "Preview includes Alpine provenance" + assert_contains "$preview_section" "skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md" "Preview includes Alpine notice" assert_not_matches "$preview_section" "\\*deleting +skills/example/agents/openai\\.yaml" "Preview preserves destination-owned OpenAI agent metadata" assert_current_branch "$dest" "$dest_branch" "Preview leaves destination checkout on its original branch" assert_branch_absent "$dest" "sync/superpowers-*" "Preview does not create sync branch in destination checkout" @@ -579,6 +858,23 @@ main() { assert_file_equals "$dirty_skill_path" "# Example Skill Locally modified fixture content." "Dirty local apply preserves tracked working-tree file content" + assert_equals "$changed_apply_status" "0" "Changed local apply exits successfully" + assert_contains "$changed_apply_output" "PR opened: https://github.com/prime-radiant-inc/openai-codex-plugins/pull/123" "Changed local apply opens PR through fake gh" + assert_contains "$changed_apply_pr_body" $'tool is behaving.\n\nVendored third-party code included in this sync' "Changed local apply PR body separates vendored section" + assert_contains "$changed_apply_pr_body" "Vendored third-party code included in this sync" "Changed local apply PR body includes vendored section" + assert_contains "$changed_apply_pr_body" "skills/brainstorming/scripts/vendor/alpine.js" "Changed local apply PR body includes vendored Alpine path" + assert_contains "$changed_apply_pr_body" "alpinejs 3.15.12" "Changed local apply PR body includes Alpine package/version" + assert_contains "$changed_apply_pr_body" "Approval artifact: SUP-215" "Changed local apply PR body includes approval artifact" + assert_contains "$changed_apply_pr_body" 'License notice: `skills/brainstorming/scripts/vendor/THIRD_PARTY_NOTICES.md`' "Changed local apply PR body includes license notice path" + assert_contains "$changed_apply_pr_body" 'Provenance: `skills/brainstorming/scripts/vendor/alpine.provenance.json`' "Changed local apply PR body includes provenance path" + assert_contains "$changed_apply_pr_body" 'SHA256: `fixture`' "Changed local apply PR body includes SHA256" + assert_file_equals "$changed_apply_alpine_path" "fixture alpine" "Changed local apply writes vendored Alpine" + assert_file_equals "$changed_apply_alpine_provenance_path" "{\"name\":\"alpinejs\",\"version\":\"3.15.12\",\"localPath\":\"skills/brainstorming/scripts/vendor/alpine.js\",\"sha256\":\"fixture\",\"approvalArtifact\":\"SUP-215\"}" "Changed local apply writes Alpine provenance" + assert_contains "$(cat "$changed_apply_alpine_notice_path")" "Alpine.js fixture notice." "Changed local apply writes Alpine notice" + assert_equals "$bootstrap_apply_status" "0" "Bootstrap local apply exits successfully" + assert_contains "$bootstrap_apply_output" "PR opened: https://github.com/prime-radiant-inc/openai-codex-plugins/pull/123" "Bootstrap local apply opens PR through fake gh" + assert_contains "$bootstrap_apply_pr_body" "Vendored third-party code included in this sync" "Bootstrap local apply PR body includes vendored section" + assert_contains "$bootstrap_apply_pr_body" "Approval artifact: SUP-215" "Bootstrap local apply PR body includes approval artifact" assert_equals "$noop_apply_status" "0" "Clean no-op local apply exits successfully" assert_contains "$noop_apply_output" "No changes — embedded plugin was already in sync with upstream" "Clean no-op local apply reports no changes" assert_current_branch "$noop_apply_dest" "$noop_apply_dest_branch" "Clean no-op local apply leaves destination checkout on its original branch" @@ -586,6 +882,9 @@ Locally modified fixture content." "Dirty local apply preserves tracked working- assert_file_equals "$noop_openai_metadata_path" "interface: display_name: \"Example\" short_description: \"Destination-owned OpenAI metadata\"" "Clean no-op local apply preserves OpenAI agent metadata" + assert_file_equals "$noop_alpine_path" "fixture alpine" "Clean no-op local apply preserves vendored Alpine" + assert_file_equals "$noop_alpine_provenance_path" "{\"name\":\"alpinejs\",\"version\":\"3.15.12\",\"localPath\":\"skills/brainstorming/scripts/vendor/alpine.js\",\"sha256\":\"fixture\",\"approvalArtifact\":\"SUP-215\"}" "Clean no-op local apply preserves Alpine provenance" + assert_contains "$(cat "$noop_alpine_notice_path")" "Alpine.js fixture notice." "Clean no-op local apply preserves Alpine notice" echo "" echo "Missing manifest assertions..." @@ -601,6 +900,7 @@ Locally modified fixture content." "Dirty local apply preserves tracked working- assert_not_contains "$script_source" "regenerated inline" "Source drops regenerated inline phrasing" assert_not_contains "$script_source" "Brand Assets directory" "Source drops Brand Assets directory phrasing" assert_not_contains "$script_source" "--assets-src" "Source drops --assets-src" + assert_contains "$script_source" "Vendored third-party code included in this sync" "Source calls out vendored third-party code in sync PR body" if [[ $FAILURES -ne 0 ]]; then echo "" diff --git a/tests/explicit-skill-requests/run-claude-describes-sdd.sh b/tests/explicit-skill-requests/run-claude-describes-sdd.sh deleted file mode 100755 index c646bc9b17..0000000000 --- a/tests/explicit-skill-requests/run-claude-describes-sdd.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -# Test where Claude explicitly describes subagent-driven-development before user requests it -# This mimics the original failure scenario - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PLUGIN_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" - -TIMESTAMP=$(date +%s) -OUTPUT_DIR="/tmp/superpowers-tests/${TIMESTAMP}/explicit-skill-requests/claude-describes" -mkdir -p "$OUTPUT_DIR" - -PROJECT_DIR="$OUTPUT_DIR/project" -mkdir -p "$PROJECT_DIR/docs/superpowers/plans" - -echo "=== Test: Claude Describes SDD First ===" -echo "Output dir: $OUTPUT_DIR" -echo "" - -cd "$PROJECT_DIR" - -# Create a plan -cat > "$PROJECT_DIR/docs/superpowers/plans/auth-system.md" << 'EOF' -# Auth System Implementation Plan - -## Task 1: Add User Model -Create user model with email and password fields. - -## Task 2: Add Auth Routes -Create login and register endpoints. - -## Task 3: Add JWT Middleware -Protect routes with JWT validation. -EOF - -# Turn 1: Have Claude describe execution options including SDD -echo ">>> Turn 1: Ask Claude to describe execution options..." -claude -p "I have a plan at docs/superpowers/plans/auth-system.md. Tell me about my options for executing it, including what subagent-driven-development means and how it works." \ - --model haiku \ - --plugin-dir "$PLUGIN_DIR" \ - --dangerously-skip-permissions \ - --max-turns 3 \ - --output-format stream-json \ - > "$OUTPUT_DIR/turn1.json" 2>&1 || true -echo "Done." - -# Turn 2: THE CRITICAL TEST - now that Claude has explained it -echo ">>> Turn 2: Request subagent-driven-development..." -FINAL_LOG="$OUTPUT_DIR/turn2.json" -claude -p "subagent-driven-development, please" \ - --continue \ - --model haiku \ - --plugin-dir "$PLUGIN_DIR" \ - --dangerously-skip-permissions \ - --max-turns 2 \ - --output-format stream-json \ - > "$FINAL_LOG" 2>&1 || true -echo "Done." -echo "" - -echo "=== Results ===" - -# Check Turn 1 to see if Claude described SDD -echo "Turn 1 - Claude's description of options (excerpt):" -grep '"type":"assistant"' "$OUTPUT_DIR/turn1.json" | head -1 | jq -r '.message.content[0].text // .message.content' 2>/dev/null | head -c 800 || echo " (could not extract)" -echo "" -echo "---" -echo "" - -# Check final turn -SKILL_PATTERN='"skill":"([^"]*:)?subagent-driven-development"' -if grep -q '"name":"Skill"' "$FINAL_LOG" && grep -qE "$SKILL_PATTERN" "$FINAL_LOG"; then - echo "PASS: Skill was triggered after Claude described it" - TRIGGERED=true -else - echo "FAIL: Skill was NOT triggered (Claude may have thought it already knew)" - TRIGGERED=false - - echo "" - echo "Tools invoked in final turn:" - grep '"type":"tool_use"' "$FINAL_LOG" | grep -o '"name":"[^"]*"' | sort -u | head -10 || echo " (none)" - - echo "" - echo "Final turn response:" - grep '"type":"assistant"' "$FINAL_LOG" | head -1 | jq -r '.message.content[0].text // .message.content' 2>/dev/null | head -c 800 || echo " (could not extract)" -fi - -echo "" -echo "Skills triggered in final turn:" -grep -o '"skill":"[^"]*"' "$FINAL_LOG" 2>/dev/null | sort -u || echo " (none)" - -echo "" -echo "Logs in: $OUTPUT_DIR" - -if [ "$TRIGGERED" = "true" ]; then - exit 0 -else - exit 1 -fi diff --git a/tests/explicit-skill-requests/run-multiturn-test.sh b/tests/explicit-skill-requests/run-multiturn-test.sh index b926d65fb3..d4c0adad2a 100755 --- a/tests/explicit-skill-requests/run-multiturn-test.sh +++ b/tests/explicit-skill-requests/run-multiturn-test.sh @@ -109,7 +109,7 @@ if [ -n "$FIRST_SKILL_LINE" ]; then PREMATURE_TOOLS=$(head -n "$FIRST_SKILL_LINE" "$TURN3_LOG" | \ grep '"type":"tool_use"' | \ grep -v '"name":"Skill"' | \ - grep -v '"name":"TodoWrite"' || true) + grep -vE '"name":"(TodoWrite|TaskCreate|TaskUpdate|TaskList|TaskGet)"' || true) if [ -n "$PREMATURE_TOOLS" ]; then echo "WARNING: Tools invoked BEFORE Skill tool in Turn 3:" echo "$PREMATURE_TOOLS" | head -5 diff --git a/tests/explicit-skill-requests/run-test.sh b/tests/explicit-skill-requests/run-test.sh index d0e7ab2951..821117c39c 100755 --- a/tests/explicit-skill-requests/run-test.sh +++ b/tests/explicit-skill-requests/run-test.sh @@ -103,11 +103,11 @@ echo "Checking for premature action..." FIRST_SKILL_LINE=$(grep -n '"name":"Skill"' "$LOG_FILE" | head -1 | cut -d: -f1) if [ -n "$FIRST_SKILL_LINE" ]; then # Check if any non-Skill, non-system tools were invoked before the first Skill invocation - # Filter out system messages, TodoWrite (planning is ok), and other non-action tools + # Filter out task tracking tools (planning is ok) and other non-action tools PREMATURE_TOOLS=$(head -n "$FIRST_SKILL_LINE" "$LOG_FILE" | \ grep '"type":"tool_use"' | \ grep -v '"name":"Skill"' | \ - grep -v '"name":"TodoWrite"' || true) + grep -vE '"name":"(TodoWrite|TaskCreate|TaskUpdate|TaskList|TaskGet)"' || true) if [ -n "$PREMATURE_TOOLS" ]; then echo "WARNING: Tools invoked BEFORE Skill tool:" echo "$PREMATURE_TOOLS" | head -5 diff --git a/tests/hooks/test-session-start.sh b/tests/hooks/test-session-start.sh new file mode 100755 index 0000000000..989d72c659 --- /dev/null +++ b/tests/hooks/test-session-start.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +HOOK_UNDER_TEST="$REPO_ROOT/hooks/session-start" +CODEX_HOOK_UNDER_TEST="$REPO_ROOT/hooks/session-start-codex" +WRAPPER_UNDER_TEST="$REPO_ROOT/hooks/run-hook.cmd" + +FAILURES=0 +TEST_ROOT="$(mktemp -d)" + +cleanup() { + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +pass() { + echo " [PASS] $1" +} + +fail() { + echo " [FAIL] $1" + FAILURES=$((FAILURES + 1)) +} + +make_home() { + local name="$1" + local home="$TEST_ROOT/$name/home" + mkdir -p "$home" + printf '%s\n' "$home" +} + +assert_command_output() { + local description="$1" + local shape="$2" + local contains="$3" + local not_contains="$4" + local home="$5" + shift 5 + + local output + if ! output="$(env -i PATH="${PATH:-}" HOME="$home" "$@" 2>&1)"; then + fail "$description" + echo " hook exited non-zero" + echo "$output" | sed 's/^/ /' + return + fi + + if printf '%s' "$output" | \ + EXPECT_SHAPE="$shape" \ + EXPECT_CONTAINS="$contains" \ + EXPECT_NOT_CONTAINS="$not_contains" \ + node -e ' +const fs = require("fs"); + +const input = fs.readFileSync(0, "utf8"); +let payload; +try { + payload = JSON.parse(input); +} catch (error) { + console.error(`invalid JSON: ${error.message}`); + process.exit(1); +} + +function hasOwn(object, key) { + return Object.prototype.hasOwnProperty.call(object, key); +} + +function fail(message) { + console.error(message); + process.exit(1); +} + +const shape = process.env.EXPECT_SHAPE; +let context; + +if (shape === "nested") { + if (!hasOwn(payload, "hookSpecificOutput")) { + fail("missing hookSpecificOutput"); + } + if (hasOwn(payload, "additional_context") || hasOwn(payload, "additionalContext")) { + fail("nested output also included a top-level context field"); + } + const hookOutput = payload.hookSpecificOutput; + if (!hookOutput || typeof hookOutput !== "object" || Array.isArray(hookOutput)) { + fail("hookSpecificOutput is not an object"); + } + if (hookOutput.hookEventName !== "SessionStart") { + fail(`unexpected hookEventName: ${hookOutput.hookEventName}`); + } + context = hookOutput.additionalContext; +} else if (shape === "cursor") { + if (hasOwn(payload, "hookSpecificOutput")) { + fail("cursor output included hookSpecificOutput"); + } + if (!hasOwn(payload, "additional_context")) { + fail("cursor output missing additional_context"); + } + if (hasOwn(payload, "additionalContext")) { + fail("cursor output included additionalContext"); + } + context = payload.additional_context; +} else if (shape === "sdk") { + if (hasOwn(payload, "hookSpecificOutput")) { + fail("sdk output included hookSpecificOutput"); + } + if (!hasOwn(payload, "additionalContext")) { + fail("sdk output missing additionalContext"); + } + if (hasOwn(payload, "additional_context")) { + fail("sdk output included additional_context"); + } + context = payload.additionalContext; +} else { + fail(`unknown expected shape: ${shape}`); +} + +if (typeof context !== "string" || context.trim() === "") { + fail("injected context was empty"); +} + +const expectedText = process.env.EXPECT_CONTAINS || ""; +if (expectedText && !context.includes(expectedText)) { + fail(`context did not contain expected text: ${expectedText}`); +} + +const forbiddenTexts = (process.env.EXPECT_NOT_CONTAINS || "") + .split("\u001f") + .filter(Boolean); +for (const forbiddenText of forbiddenTexts) { + if (context.includes(forbiddenText)) { + fail(`context contained forbidden text: ${forbiddenText}`); + } +} +'; then + pass "$description" + else + fail "$description" + echo " output:" + echo "$output" | sed 's/^/ /' + fi +} + +echo "SessionStart hook output tests" + +claude_home="$(make_home claude-code)" +assert_command_output \ + "Claude Code emits nested SessionStart additionalContext" \ + "nested" \ + "" \ + "" \ + "$claude_home" \ + CLAUDE_PLUGIN_ROOT="$REPO_ROOT" \ + bash "$HOOK_UNDER_TEST" + +codex_home="$(make_home codex-plugin-hooks)" +codex_data="$TEST_ROOT/codex-plugin-hooks/data" +mkdir -p "$codex_data" +assert_command_output \ + "Codex plugin hooks use dedicated script and emit nested SessionStart additionalContext" \ + "nested" \ + "" \ + "" \ + "$codex_home" \ + PLUGIN_DATA="$codex_data" \ + CLAUDE_PLUGIN_DATA="$codex_data" \ + PLUGIN_ROOT="$REPO_ROOT" \ + CLAUDE_PLUGIN_ROOT="$REPO_ROOT" \ + bash "$CODEX_HOOK_UNDER_TEST" + +codex_wrapper_home="$(make_home codex-wrapper)" +codex_wrapper_data="$TEST_ROOT/codex-wrapper/data" +mkdir -p "$codex_wrapper_data" +assert_command_output \ + "Codex wrapper path dispatches to dedicated script" \ + "nested" \ + "" \ + "" \ + "$codex_wrapper_home" \ + PLUGIN_DATA="$codex_wrapper_data" \ + CLAUDE_PLUGIN_DATA="$codex_wrapper_data" \ + PLUGIN_ROOT="$REPO_ROOT" \ + CLAUDE_PLUGIN_ROOT="$REPO_ROOT" \ + bash "$WRAPPER_UNDER_TEST" session-start-codex + +cursor_home="$(make_home cursor)" +assert_command_output \ + "Cursor emits top-level additional_context only" \ + "cursor" \ + "" \ + "" \ + "$cursor_home" \ + CURSOR_PLUGIN_ROOT="$REPO_ROOT" \ + CLAUDE_PLUGIN_ROOT="$REPO_ROOT" \ + bash "$HOOK_UNDER_TEST" + +copilot_home="$(make_home copilot-cli)" +assert_command_output \ + "Copilot CLI emits top-level additionalContext only" \ + "sdk" \ + "" \ + "" \ + "$copilot_home" \ + COPILOT_CLI=1 \ + CLAUDE_PLUGIN_ROOT="$REPO_ROOT" \ + bash "$HOOK_UNDER_TEST" + +legacy_home="$(make_home legacy-warning-removed)" +mkdir -p "$legacy_home/.config/superpowers/skills" +assert_command_output \ + "SessionStart omits obsolete legacy custom-skill warning" \ + "nested" \ + "" \ + "Superpowers now uses"$'\037'"~/.config/superpowers/skills"$'\037'"~/.claude/skills"$'\037'"legacy" \ + "$legacy_home" \ + CLAUDE_PLUGIN_ROOT="$REPO_ROOT" \ + bash "$HOOK_UNDER_TEST" + +codex_legacy_home="$(make_home codex-legacy-warning-removed)" +codex_legacy_data="$TEST_ROOT/codex-legacy-warning-removed/data" +mkdir -p "$codex_legacy_home/.config/superpowers/skills" "$codex_legacy_data" +assert_command_output \ + "Codex SessionStart omits obsolete legacy custom-skill warning" \ + "nested" \ + "" \ + "Superpowers now uses"$'\037'"~/.config/superpowers/skills"$'\037'"~/.claude/skills"$'\037'"legacy" \ + "$codex_legacy_home" \ + PLUGIN_DATA="$codex_legacy_data" \ + CLAUDE_PLUGIN_DATA="$codex_legacy_data" \ + PLUGIN_ROOT="$REPO_ROOT" \ + CLAUDE_PLUGIN_ROOT="$REPO_ROOT" \ + bash "$CODEX_HOOK_UNDER_TEST" + +if [[ "$FAILURES" -gt 0 ]]; then + echo "STATUS: FAILED ($FAILURES failure(s))" + exit 1 +fi + +echo "STATUS: PASSED" diff --git a/tests/opencode/test-bootstrap-caching.mjs b/tests/opencode/test-bootstrap-caching.mjs index 55c4e9eb31..32149aed33 100644 --- a/tests/opencode/test-bootstrap-caching.mjs +++ b/tests/opencode/test-bootstrap-caching.mjs @@ -44,6 +44,10 @@ const result = { scenario, firstBootstrapParts: countBootstrapParts(firstOutput), secondBootstrapParts: countBootstrapParts(secondOutput), + staleMentionMapping: bootstrapText(firstOutput).includes('@mention'), + staleTaskMapping: bootstrapText(firstOutput).includes('`Task` tool with subagents'), + mapsSubagentToTask: bootstrapText(firstOutput).includes('`task` with `subagent_type: "general"`'), + mapsMutationToApplyPatch: bootstrapText(firstOutput).includes('`apply_patch`'), firstReadCount: afterFirst.readCount, secondReadCount: afterSecond.readCount, firstExistsCount: afterFirst.existsCount, @@ -83,6 +87,12 @@ function countBootstrapParts(output) { ).length; } +function bootstrapText(output) { + return output.messages[0].parts.find( + (part) => part.type === 'text' && part.text.includes('EXTREMELY_IMPORTANT') + )?.text || ''; +} + function assertPresentBootstrap(result) { const failures = []; if (result.firstBootstrapParts !== 1) { @@ -100,6 +110,18 @@ function assertPresentBootstrap(result) { if (result.secondExistsCount !== result.firstExistsCount) { failures.push(`expected cached second transform to do no additional exists checks, got ${result.secondExistsCount - result.firstExistsCount}`); } + if (result.staleMentionMapping) { + failures.push('expected OpenCode bootstrap not to teach @mention subagent syntax'); + } + if (result.staleTaskMapping) { + failures.push('expected OpenCode bootstrap not to teach stale Task-tool mapping'); + } + if (!result.mapsSubagentToTask) { + failures.push('expected OpenCode bootstrap to map general-purpose subagents to task with subagent_type'); + } + if (!result.mapsMutationToApplyPatch) { + failures.push('expected OpenCode bootstrap to map file mutation to apply_patch'); + } return failures; } diff --git a/tests/pi/test-pi-extension.mjs b/tests/pi/test-pi-extension.mjs new file mode 100644 index 0000000000..196e975934 --- /dev/null +++ b/tests/pi/test-pi-extension.mjs @@ -0,0 +1,128 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import test from 'node:test'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(__dirname, '../..'); +const packageJsonPath = resolve(repoRoot, 'package.json'); +const extensionPath = resolve(repoRoot, '.pi/extensions/superpowers.ts'); +const piToolsPath = resolve(repoRoot, 'skills/using-superpowers/references/pi-tools.md'); + +async function readPackageJson() { + return JSON.parse(await readFile(packageJsonPath, 'utf8')); +} + +async function loadExtension() { + const handlers = new Map(); + const pi = { + on(event, handler) { + if (!handlers.has(event)) handlers.set(event, []); + handlers.get(event).push(handler); + }, + }; + const mod = await import(pathToFileURL(extensionPath).href + `?cachebust=${Date.now()}-${Math.random()}`); + mod.default(pi); + return { handlers }; +} + +function firstHandler(handlers, event) { + const eventHandlers = handlers.get(event) ?? []; + assert.equal(eventHandlers.length, 1, `expected one ${event} handler`); + return eventHandlers[0]; +} + +function textOf(message) { + if (typeof message.content === 'string') return message.content; + return message.content + .filter((part) => part.type === 'text') + .map((part) => part.text) + .join('\n'); +} + +test('package.json declares a pi package with skills and extension resources', async () => { + const pkg = await readPackageJson(); + + assert.equal(pkg.name, 'superpowers'); + assert.ok(pkg.keywords.includes('pi-package')); + assert.deepEqual(pkg.pi.skills, ['./skills']); + assert.deepEqual(pkg.pi.extensions, ['./.pi/extensions/superpowers.ts']); +}); + +test('extension registers lifecycle hooks without pre-compaction injection', async () => { + const { handlers } = await loadExtension(); + + for (const event of ['resources_discover', 'session_start', 'session_compact', 'context', 'agent_end']) { + assert.equal((handlers.get(event) ?? []).length, 1, `missing ${event} handler`); + } + assert.equal((handlers.get('session_before_compact') ?? []).length, 0); +}); + +test('resources_discover contributes the bundled skills directory', async () => { + const { handlers } = await loadExtension(); + const discover = firstHandler(handlers, 'resources_discover'); + + const result = await discover({ type: 'resources_discover', cwd: repoRoot, reason: 'startup' }, {}); + + assert.deepEqual(result.skillPaths, [resolve(repoRoot, 'skills')]); +}); + +test('startup context injects the bootstrap as one user message until agent_end', async () => { + const { handlers } = await loadExtension(); + const sessionStart = firstHandler(handlers, 'session_start'); + const context = firstHandler(handlers, 'context'); + const agentEnd = firstHandler(handlers, 'agent_end'); + + await sessionStart({ type: 'session_start', reason: 'startup' }, {}); + + const originalMessages = [ + { role: 'user', content: [{ type: 'text', text: 'Let us make a react todo list' }], timestamp: 1 }, + ]; + const result = await context({ type: 'context', messages: originalMessages }, {}); + + assert.equal(result.messages.length, 2); + assert.equal(result.messages[0].role, 'user'); + assert.match(textOf(result.messages[0]), /You have superpowers/); + assert.match(textOf(result.messages[0]), /Pi tool mapping/); + assert.equal(result.messages[1], originalMessages[0]); + + const repeatedProviderRequest = await context({ type: 'context', messages: originalMessages }, {}); + assert.equal(repeatedProviderRequest.messages.length, 2); + assert.match(textOf(repeatedProviderRequest.messages[0]), /You have superpowers/); + + const alreadyInjected = await context({ type: 'context', messages: result.messages }, {}); + assert.equal(alreadyInjected, undefined, 'bootstrap should not duplicate when already present'); + + await agentEnd({ type: 'agent_end', messages: [] }, {}); + const afterEnd = await context({ type: 'context', messages: originalMessages }, {}); + assert.equal(afterEnd, undefined, 'startup bootstrap should clear after agent_end'); +}); + +test('session_compact injects bootstrap after compaction summaries, not before compaction', async () => { + const { handlers } = await loadExtension(); + const sessionCompact = firstHandler(handlers, 'session_compact'); + const context = firstHandler(handlers, 'context'); + + await sessionCompact({ type: 'session_compact', compactionEntry: {}, fromExtension: false }, {}); + + const summary = { role: 'compactionSummary', summary: 'Prior work summary', tokensBefore: 123, timestamp: 1 }; + const user = { role: 'user', content: [{ type: 'text', text: 'Continue' }], timestamp: 2 }; + const result = await context({ type: 'context', messages: [summary, user] }, {}); + + assert.equal(result.messages.length, 3); + assert.equal(result.messages[0], summary); + assert.equal(result.messages[1].role, 'user'); + assert.match(textOf(result.messages[1]), /You have superpowers/); + assert.equal(result.messages[2], user); +}); + +test('pi tools reference documents pi-specific mappings', async () => { + assert.equal(existsSync(piToolsPath), true, 'pi-tools.md should exist'); + const text = await readFile(piToolsPath, 'utf8'); + + for (const expected of ['Skill', 'Task', 'TodoWrite', 'read', 'write', 'edit', 'bash']) { + assert.match(text, new RegExp(expected)); + } +}); diff --git a/tests/skill-triggering/prompts/dispatching-parallel-agents.txt b/tests/skill-triggering/prompts/dispatching-parallel-agents.txt deleted file mode 100644 index fb5423f290..0000000000 --- a/tests/skill-triggering/prompts/dispatching-parallel-agents.txt +++ /dev/null @@ -1,8 +0,0 @@ -I have 4 independent test failures happening in different modules: - -1. tests/auth/login.test.ts - "should redirect after login" is failing -2. tests/api/users.test.ts - "should return user list" returns 500 -3. tests/components/Button.test.tsx - snapshot mismatch -4. tests/utils/date.test.ts - timezone handling broken - -These are unrelated issues in different parts of the codebase. Can you investigate all of them? \ No newline at end of file diff --git a/tests/skill-triggering/prompts/executing-plans.txt b/tests/skill-triggering/prompts/executing-plans.txt deleted file mode 100644 index 86ed26257c..0000000000 --- a/tests/skill-triggering/prompts/executing-plans.txt +++ /dev/null @@ -1 +0,0 @@ -I have a plan document at docs/superpowers/plans/2024-01-15-auth-system.md that needs to be executed. Please implement it. \ No newline at end of file diff --git a/tests/skill-triggering/prompts/requesting-code-review.txt b/tests/skill-triggering/prompts/requesting-code-review.txt deleted file mode 100644 index f1be2672a1..0000000000 --- a/tests/skill-triggering/prompts/requesting-code-review.txt +++ /dev/null @@ -1,3 +0,0 @@ -I just finished implementing the user authentication feature. All the code is committed. Can you review the changes before I merge to main? - -The commits are between abc123 and def456. \ No newline at end of file diff --git a/tests/skill-triggering/prompts/systematic-debugging.txt b/tests/skill-triggering/prompts/systematic-debugging.txt deleted file mode 100644 index d3806b9c26..0000000000 --- a/tests/skill-triggering/prompts/systematic-debugging.txt +++ /dev/null @@ -1,11 +0,0 @@ -The tests are failing with this error: - -``` -FAIL src/utils/parser.test.ts - ● Parser › should handle nested objects - TypeError: Cannot read property 'value' of undefined - at parse (src/utils/parser.ts:42:18) - at Object. (src/utils/parser.test.ts:28:20) -``` - -Can you figure out what's going wrong and fix it? \ No newline at end of file diff --git a/tests/skill-triggering/prompts/test-driven-development.txt b/tests/skill-triggering/prompts/test-driven-development.txt deleted file mode 100644 index f386eeab0a..0000000000 --- a/tests/skill-triggering/prompts/test-driven-development.txt +++ /dev/null @@ -1,7 +0,0 @@ -I need to add a new feature to validate email addresses. It should: -- Check that there's an @ symbol -- Check that there's at least one character before the @ -- Check that there's a dot in the domain part -- Return true/false - -Can you implement this? \ No newline at end of file diff --git a/tests/skill-triggering/prompts/writing-plans.txt b/tests/skill-triggering/prompts/writing-plans.txt deleted file mode 100644 index 74803133a9..0000000000 --- a/tests/skill-triggering/prompts/writing-plans.txt +++ /dev/null @@ -1,10 +0,0 @@ -Here's the spec for our new authentication system: - -Requirements: -- Users can register with email/password -- Users can log in and receive a JWT token -- Protected routes require valid JWT -- Tokens expire after 24 hours -- Support password reset via email - -We need to implement this. There are multiple steps involved - user model, auth routes, middleware, email service integration. \ No newline at end of file diff --git a/tests/skill-triggering/run-all.sh b/tests/skill-triggering/run-all.sh deleted file mode 100755 index 1a35dd9326..0000000000 --- a/tests/skill-triggering/run-all.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# Run all skill triggering tests -# Usage: ./run-all.sh - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROMPTS_DIR="$SCRIPT_DIR/prompts" - -SKILLS=( - "systematic-debugging" - "test-driven-development" - "writing-plans" - "dispatching-parallel-agents" - "executing-plans" - "requesting-code-review" -) - -echo "=== Running Skill Triggering Tests ===" -echo "" - -PASSED=0 -FAILED=0 -RESULTS=() - -for skill in "${SKILLS[@]}"; do - prompt_file="$PROMPTS_DIR/${skill}.txt" - - if [ ! -f "$prompt_file" ]; then - echo "⚠️ SKIP: No prompt file for $skill" - continue - fi - - echo "Testing: $skill" - - if "$SCRIPT_DIR/run-test.sh" "$skill" "$prompt_file" 3 2>&1 | tee /tmp/skill-test-$skill.log; then - PASSED=$((PASSED + 1)) - RESULTS+=("✅ $skill") - else - FAILED=$((FAILED + 1)) - RESULTS+=("❌ $skill") - fi - - echo "" - echo "---" - echo "" -done - -echo "" -echo "=== Summary ===" -for result in "${RESULTS[@]}"; do - echo " $result" -done -echo "" -echo "Passed: $PASSED" -echo "Failed: $FAILED" - -if [ $FAILED -gt 0 ]; then - exit 1 -fi diff --git a/tests/skill-triggering/run-test.sh b/tests/skill-triggering/run-test.sh deleted file mode 100755 index ba9199583e..0000000000 --- a/tests/skill-triggering/run-test.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -# Test skill triggering with naive prompts -# Usage: ./run-test.sh -# -# Tests whether Claude triggers a skill based on a natural prompt -# (without explicitly mentioning the skill) - -set -e - -SKILL_NAME="$1" -PROMPT_FILE="$2" -MAX_TURNS="${3:-3}" - -if [ -z "$SKILL_NAME" ] || [ -z "$PROMPT_FILE" ]; then - echo "Usage: $0 [max-turns]" - echo "Example: $0 systematic-debugging ./test-prompts/debugging.txt" - exit 1 -fi - -# Get the directory where this script lives (should be tests/skill-triggering) -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Get the superpowers plugin root (two levels up from tests/skill-triggering) -PLUGIN_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" - -TIMESTAMP=$(date +%s) -OUTPUT_DIR="/tmp/superpowers-tests/${TIMESTAMP}/skill-triggering/${SKILL_NAME}" -mkdir -p "$OUTPUT_DIR" - -# Read prompt from file -PROMPT=$(cat "$PROMPT_FILE") - -echo "=== Skill Triggering Test ===" -echo "Skill: $SKILL_NAME" -echo "Prompt file: $PROMPT_FILE" -echo "Max turns: $MAX_TURNS" -echo "Output dir: $OUTPUT_DIR" -echo "" - -# Copy prompt for reference -cp "$PROMPT_FILE" "$OUTPUT_DIR/prompt.txt" - -# Run Claude -LOG_FILE="$OUTPUT_DIR/claude-output.json" -cd "$OUTPUT_DIR" - -echo "Plugin dir: $PLUGIN_DIR" -echo "Running claude -p with naive prompt..." -timeout 300 claude -p "$PROMPT" \ - --plugin-dir "$PLUGIN_DIR" \ - --dangerously-skip-permissions \ - --max-turns "$MAX_TURNS" \ - --output-format stream-json \ - > "$LOG_FILE" 2>&1 || true - -echo "" -echo "=== Results ===" - -# Check if skill was triggered (look for Skill tool invocation) -# In stream-json, tool invocations have "name":"Skill" (not "tool":"Skill") -# Match either "skill":"skillname" or "skill":"namespace:skillname" -SKILL_PATTERN='"skill":"([^"]*:)?'"${SKILL_NAME}"'"' -if grep -q '"name":"Skill"' "$LOG_FILE" && grep -qE "$SKILL_PATTERN" "$LOG_FILE"; then - echo "✅ PASS: Skill '$SKILL_NAME' was triggered" - TRIGGERED=true -else - echo "❌ FAIL: Skill '$SKILL_NAME' was NOT triggered" - TRIGGERED=false -fi - -# Show what skills WERE triggered -echo "" -echo "Skills triggered in this run:" -grep -o '"skill":"[^"]*"' "$LOG_FILE" 2>/dev/null | sort -u || echo " (none)" - -# Show first assistant message -echo "" -echo "First assistant response (truncated):" -grep '"type":"assistant"' "$LOG_FILE" | head -1 | jq -r '.message.content[0].text // .message.content' 2>/dev/null | head -c 500 || echo " (could not extract)" - -echo "" -echo "Full log: $LOG_FILE" -echo "Timestamp: $TIMESTAMP" - -if [ "$TRIGGERED" = "true" ]; then - exit 0 -else - exit 1 -fi diff --git a/tests/subagent-driven-dev/go-fractals/design.md b/tests/subagent-driven-dev/go-fractals/design.md deleted file mode 100644 index 2fbc6b1f40..0000000000 --- a/tests/subagent-driven-dev/go-fractals/design.md +++ /dev/null @@ -1,81 +0,0 @@ -# Go Fractals CLI - Design - -## Overview - -A command-line tool that generates ASCII art fractals. Supports two fractal types with configurable output. - -## Usage - -```bash -# Sierpinski triangle -fractals sierpinski --size 32 --depth 5 - -# Mandelbrot set -fractals mandelbrot --width 80 --height 24 --iterations 100 - -# Custom character -fractals sierpinski --size 16 --char '#' - -# Help -fractals --help -fractals sierpinski --help -``` - -## Commands - -### `sierpinski` - -Generates a Sierpinski triangle using recursive subdivision. - -Flags: -- `--size` (default: 32) - Width of the triangle base in characters -- `--depth` (default: 5) - Recursion depth -- `--char` (default: '*') - Character to use for filled points - -Output: Triangle printed to stdout, one line per row. - -### `mandelbrot` - -Renders the Mandelbrot set as ASCII art. Maps iteration count to characters. - -Flags: -- `--width` (default: 80) - Output width in characters -- `--height` (default: 24) - Output height in characters -- `--iterations` (default: 100) - Maximum iterations for escape calculation -- `--char` (default: gradient) - Single character, or omit for gradient " .:-=+*#%@" - -Output: Rectangle printed to stdout. - -## Architecture - -``` -cmd/ - fractals/ - main.go # Entry point, CLI setup -internal/ - sierpinski/ - sierpinski.go # Algorithm - sierpinski_test.go - mandelbrot/ - mandelbrot.go # Algorithm - mandelbrot_test.go - cli/ - root.go # Root command, help - sierpinski.go # Sierpinski subcommand - mandelbrot.go # Mandelbrot subcommand -``` - -## Dependencies - -- Go 1.21+ -- `github.com/spf13/cobra` for CLI - -## Acceptance Criteria - -1. `fractals --help` shows usage -2. `fractals sierpinski` outputs a recognizable triangle -3. `fractals mandelbrot` outputs a recognizable Mandelbrot set -4. `--size`, `--width`, `--height`, `--depth`, `--iterations` flags work -5. `--char` customizes output character -6. Invalid inputs produce clear error messages -7. All tests pass diff --git a/tests/subagent-driven-dev/go-fractals/plan.md b/tests/subagent-driven-dev/go-fractals/plan.md deleted file mode 100644 index 9875ab5f23..0000000000 --- a/tests/subagent-driven-dev/go-fractals/plan.md +++ /dev/null @@ -1,172 +0,0 @@ -# Go Fractals CLI - Implementation Plan - -Execute this plan using the `superpowers:subagent-driven-development` skill. - -## Context - -Building a CLI tool that generates ASCII fractals. See `design.md` for full specification. - -## Tasks - -### Task 1: Project Setup - -Create the Go module and directory structure. - -**Do:** -- Initialize `go.mod` with module name `github.com/superpowers-test/fractals` -- Create directory structure: `cmd/fractals/`, `internal/sierpinski/`, `internal/mandelbrot/`, `internal/cli/` -- Create minimal `cmd/fractals/main.go` that prints "fractals cli" -- Add `github.com/spf13/cobra` dependency - -**Verify:** -- `go build ./cmd/fractals` succeeds -- `./fractals` prints "fractals cli" - ---- - -### Task 2: CLI Framework with Help - -Set up Cobra root command with help output. - -**Do:** -- Create `internal/cli/root.go` with root command -- Configure help text showing available subcommands -- Wire root command into `main.go` - -**Verify:** -- `./fractals --help` shows usage with "sierpinski" and "mandelbrot" listed as available commands -- `./fractals` (no args) shows help - ---- - -### Task 3: Sierpinski Algorithm - -Implement the Sierpinski triangle generation algorithm. - -**Do:** -- Create `internal/sierpinski/sierpinski.go` -- Implement `Generate(size, depth int, char rune) []string` that returns lines of the triangle -- Use recursive midpoint subdivision algorithm -- Create `internal/sierpinski/sierpinski_test.go` with tests: - - Small triangle (size=4, depth=2) matches expected output - - Size=1 returns single character - - Depth=0 returns filled triangle - -**Verify:** -- `go test ./internal/sierpinski/...` passes - ---- - -### Task 4: Sierpinski CLI Integration - -Wire the Sierpinski algorithm to a CLI subcommand. - -**Do:** -- Create `internal/cli/sierpinski.go` with `sierpinski` subcommand -- Add flags: `--size` (default 32), `--depth` (default 5), `--char` (default '*') -- Call `sierpinski.Generate()` and print result to stdout - -**Verify:** -- `./fractals sierpinski` outputs a triangle -- `./fractals sierpinski --size 16 --depth 3` outputs smaller triangle -- `./fractals sierpinski --help` shows flag documentation - ---- - -### Task 5: Mandelbrot Algorithm - -Implement the Mandelbrot set ASCII renderer. - -**Do:** -- Create `internal/mandelbrot/mandelbrot.go` -- Implement `Render(width, height, maxIter int, char string) []string` -- Map complex plane region (-2.5 to 1.0 real, -1.0 to 1.0 imaginary) to output dimensions -- Map iteration count to character gradient " .:-=+*#%@" (or single char if provided) -- Create `internal/mandelbrot/mandelbrot_test.go` with tests: - - Output dimensions match requested width/height - - Known point inside set (0,0) maps to max-iteration character - - Known point outside set (2,0) maps to low-iteration character - -**Verify:** -- `go test ./internal/mandelbrot/...` passes - ---- - -### Task 6: Mandelbrot CLI Integration - -Wire the Mandelbrot algorithm to a CLI subcommand. - -**Do:** -- Create `internal/cli/mandelbrot.go` with `mandelbrot` subcommand -- Add flags: `--width` (default 80), `--height` (default 24), `--iterations` (default 100), `--char` (default "") -- Call `mandelbrot.Render()` and print result to stdout - -**Verify:** -- `./fractals mandelbrot` outputs recognizable Mandelbrot set -- `./fractals mandelbrot --width 40 --height 12` outputs smaller version -- `./fractals mandelbrot --help` shows flag documentation - ---- - -### Task 7: Character Set Configuration - -Ensure `--char` flag works consistently across both commands. - -**Do:** -- Verify Sierpinski `--char` flag passes character to algorithm -- For Mandelbrot, `--char` should use single character instead of gradient -- Add tests for custom character output - -**Verify:** -- `./fractals sierpinski --char '#'` uses '#' character -- `./fractals mandelbrot --char '.'` uses '.' for all filled points -- Tests pass - ---- - -### Task 8: Input Validation and Error Handling - -Add validation for invalid inputs. - -**Do:** -- Sierpinski: size must be > 0, depth must be >= 0 -- Mandelbrot: width/height must be > 0, iterations must be > 0 -- Return clear error messages for invalid inputs -- Add tests for error cases - -**Verify:** -- `./fractals sierpinski --size 0` prints error, exits non-zero -- `./fractals mandelbrot --width -1` prints error, exits non-zero -- Error messages are clear and helpful - ---- - -### Task 9: Integration Tests - -Add integration tests that invoke the CLI. - -**Do:** -- Create `cmd/fractals/main_test.go` or `test/integration_test.go` -- Test full CLI invocation for both commands -- Verify output format and exit codes -- Test error cases return non-zero exit - -**Verify:** -- `go test ./...` passes all tests including integration tests - ---- - -### Task 10: README - -Document usage and examples. - -**Do:** -- Create `README.md` with: - - Project description - - Installation: `go install ./cmd/fractals` - - Usage examples for both commands - - Example output (small samples) - -**Verify:** -- README accurately describes the tool -- Examples in README actually work diff --git a/tests/subagent-driven-dev/go-fractals/scaffold.sh b/tests/subagent-driven-dev/go-fractals/scaffold.sh deleted file mode 100755 index 646a6153db..0000000000 --- a/tests/subagent-driven-dev/go-fractals/scaffold.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# Scaffold the Go Fractals test project -# Usage: ./scaffold.sh /path/to/target/directory - -set -e - -TARGET_DIR="${1:?Usage: $0 }" -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -# Create target directory -mkdir -p "$TARGET_DIR" -cd "$TARGET_DIR" - -# Initialize git repo -git init - -# Copy design and plan -cp "$SCRIPT_DIR/design.md" . -cp "$SCRIPT_DIR/plan.md" . - -# Create .claude settings to allow reads/writes in this directory -mkdir -p .claude -cat > .claude/settings.local.json << 'SETTINGS' -{ - "permissions": { - "allow": [ - "Read(**)", - "Edit(**)", - "Write(**)", - "Bash(go:*)", - "Bash(mkdir:*)", - "Bash(git:*)" - ] - } -} -SETTINGS - -# Create initial commit -git add . -git commit -m "Initial project setup with design and plan" - -echo "Scaffolded Go Fractals project at: $TARGET_DIR" -echo "" -echo "To run the test:" -echo " claude -p \"Execute this plan using superpowers:subagent-driven-development. Plan: $TARGET_DIR/plan.md\" --plugin-dir /path/to/superpowers" diff --git a/tests/subagent-driven-dev/run-test.sh b/tests/subagent-driven-dev/run-test.sh deleted file mode 100755 index 807cb2df5a..0000000000 --- a/tests/subagent-driven-dev/run-test.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash -# Run a subagent-driven-development test -# Usage: ./run-test.sh [--plugin-dir ] -# -# Example: -# ./run-test.sh go-fractals -# ./run-test.sh svelte-todo --plugin-dir /path/to/superpowers - -set -e - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -TEST_NAME="${1:?Usage: $0 [--plugin-dir ]}" -shift - -# Parse optional arguments -PLUGIN_DIR="" -while [[ $# -gt 0 ]]; do - case $1 in - --plugin-dir) - PLUGIN_DIR="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" - exit 1 - ;; - esac -done - -# Default plugin dir to parent of tests directory -if [[ -z "$PLUGIN_DIR" ]]; then - PLUGIN_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" -fi - -# Verify test exists -TEST_DIR="$SCRIPT_DIR/$TEST_NAME" -if [[ ! -d "$TEST_DIR" ]]; then - echo "Error: Test '$TEST_NAME' not found at $TEST_DIR" - echo "Available tests:" - ls -1 "$SCRIPT_DIR" | grep -v '\.sh$' | grep -v '\.md$' - exit 1 -fi - -# Create timestamped output directory -TIMESTAMP=$(date +%s) -OUTPUT_BASE="/tmp/superpowers-tests/$TIMESTAMP/subagent-driven-development" -OUTPUT_DIR="$OUTPUT_BASE/$TEST_NAME" -mkdir -p "$OUTPUT_DIR" - -echo "=== Subagent-Driven Development Test ===" -echo "Test: $TEST_NAME" -echo "Output: $OUTPUT_DIR" -echo "Plugin: $PLUGIN_DIR" -echo "" - -# Scaffold the project -echo ">>> Scaffolding project..." -"$TEST_DIR/scaffold.sh" "$OUTPUT_DIR/project" -echo "" - -# Prepare the prompt -PLAN_PATH="$OUTPUT_DIR/project/plan.md" -PROMPT="Execute this plan using superpowers:subagent-driven-development. The plan is at: $PLAN_PATH" - -# Run Claude with JSON output for token tracking -LOG_FILE="$OUTPUT_DIR/claude-output.json" -echo ">>> Running Claude..." -echo "Prompt: $PROMPT" -echo "Log file: $LOG_FILE" -echo "" - -# Run claude and capture output -# Using stream-json to get token usage stats -# --dangerously-skip-permissions for automated testing (subagents don't inherit parent settings) -cd "$OUTPUT_DIR/project" -claude -p "$PROMPT" \ - --plugin-dir "$PLUGIN_DIR" \ - --dangerously-skip-permissions \ - --output-format stream-json \ - --verbose \ - > "$LOG_FILE" 2>&1 || true - -# Extract final stats -echo "" -echo ">>> Test complete" -echo "Project directory: $OUTPUT_DIR/project" -echo "Claude log: $LOG_FILE" -echo "" - -# Show token usage if available -if command -v jq &> /dev/null; then - echo ">>> Token usage:" - # Extract usage from the last message with usage info - jq -s '[.[] | select(.type == "result")] | last | .usage' "$LOG_FILE" 2>/dev/null || echo "(could not parse usage)" - echo "" -fi - -echo ">>> Next steps:" -echo "1. Review the project: cd $OUTPUT_DIR/project" -echo "2. Review Claude's log: less $LOG_FILE" -echo "3. Check if tests pass:" -if [[ "$TEST_NAME" == "go-fractals" ]]; then - echo " cd $OUTPUT_DIR/project && go test ./..." -elif [[ "$TEST_NAME" == "svelte-todo" ]]; then - echo " cd $OUTPUT_DIR/project && npm test && npx playwright test" -fi diff --git a/tests/subagent-driven-dev/svelte-todo/design.md b/tests/subagent-driven-dev/svelte-todo/design.md deleted file mode 100644 index ccbb10fe5c..0000000000 --- a/tests/subagent-driven-dev/svelte-todo/design.md +++ /dev/null @@ -1,70 +0,0 @@ -# Svelte Todo List - Design - -## Overview - -A simple todo list application built with Svelte. Supports creating, completing, and deleting todos with localStorage persistence. - -## Features - -- Add new todos -- Mark todos as complete/incomplete -- Delete todos -- Filter by: All / Active / Completed -- Clear all completed todos -- Persist to localStorage -- Show count of remaining items - -## User Interface - -``` -┌─────────────────────────────────────────┐ -│ Svelte Todos │ -├─────────────────────────────────────────┤ -│ [________________________] [Add] │ -├─────────────────────────────────────────┤ -│ [ ] Buy groceries [x] │ -│ [✓] Walk the dog [x] │ -│ [ ] Write code [x] │ -├─────────────────────────────────────────┤ -│ 2 items left │ -│ [All] [Active] [Completed] [Clear ✓] │ -└─────────────────────────────────────────┘ -``` - -## Components - -``` -src/ - App.svelte # Main app, state management - lib/ - TodoInput.svelte # Text input + Add button - TodoList.svelte # List container - TodoItem.svelte # Single todo with checkbox, text, delete - FilterBar.svelte # Filter buttons + clear completed - store.ts # Svelte store for todos - storage.ts # localStorage persistence -``` - -## Data Model - -```typescript -interface Todo { - id: string; // UUID - text: string; // Todo text - completed: boolean; -} - -type Filter = 'all' | 'active' | 'completed'; -``` - -## Acceptance Criteria - -1. Can add a todo by typing and pressing Enter or clicking Add -2. Can toggle todo completion by clicking checkbox -3. Can delete a todo by clicking X button -4. Filter buttons show correct subset of todos -5. "X items left" shows count of incomplete todos -6. "Clear completed" removes all completed todos -7. Todos persist across page refresh (localStorage) -8. Empty state shows helpful message -9. All tests pass diff --git a/tests/subagent-driven-dev/svelte-todo/plan.md b/tests/subagent-driven-dev/svelte-todo/plan.md deleted file mode 100644 index f4e555b305..0000000000 --- a/tests/subagent-driven-dev/svelte-todo/plan.md +++ /dev/null @@ -1,222 +0,0 @@ -# Svelte Todo List - Implementation Plan - -Execute this plan using the `superpowers:subagent-driven-development` skill. - -## Context - -Building a todo list app with Svelte. See `design.md` for full specification. - -## Tasks - -### Task 1: Project Setup - -Create the Svelte project with Vite. - -**Do:** -- Run `npm create vite@latest . -- --template svelte-ts` -- Install dependencies with `npm install` -- Verify dev server works -- Clean up default Vite template content from App.svelte - -**Verify:** -- `npm run dev` starts server -- App shows minimal "Svelte Todos" heading -- `npm run build` succeeds - ---- - -### Task 2: Todo Store - -Create the Svelte store for todo state management. - -**Do:** -- Create `src/lib/store.ts` -- Define `Todo` interface with id, text, completed -- Create writable store with initial empty array -- Export functions: `addTodo(text)`, `toggleTodo(id)`, `deleteTodo(id)`, `clearCompleted()` -- Create `src/lib/store.test.ts` with tests for each function - -**Verify:** -- Tests pass: `npm run test` (install vitest if needed) - ---- - -### Task 3: localStorage Persistence - -Add persistence layer for todos. - -**Do:** -- Create `src/lib/storage.ts` -- Implement `loadTodos(): Todo[]` and `saveTodos(todos: Todo[])` -- Handle JSON parse errors gracefully (return empty array) -- Integrate with store: load on init, save on change -- Add tests for load/save/error handling - -**Verify:** -- Tests pass -- Manual test: add todo, refresh page, todo persists - ---- - -### Task 4: TodoInput Component - -Create the input component for adding todos. - -**Do:** -- Create `src/lib/TodoInput.svelte` -- Text input bound to local state -- Add button calls `addTodo()` and clears input -- Enter key also submits -- Disable Add button when input is empty -- Add component tests - -**Verify:** -- Tests pass -- Component renders input and button - ---- - -### Task 5: TodoItem Component - -Create the single todo item component. - -**Do:** -- Create `src/lib/TodoItem.svelte` -- Props: `todo: Todo` -- Checkbox toggles completion (calls `toggleTodo`) -- Text with strikethrough when completed -- Delete button (X) calls `deleteTodo` -- Add component tests - -**Verify:** -- Tests pass -- Component renders checkbox, text, delete button - ---- - -### Task 6: TodoList Component - -Create the list container component. - -**Do:** -- Create `src/lib/TodoList.svelte` -- Props: `todos: Todo[]` -- Renders TodoItem for each todo -- Shows "No todos yet" when empty -- Add component tests - -**Verify:** -- Tests pass -- Component renders list of TodoItems - ---- - -### Task 7: FilterBar Component - -Create the filter and status bar component. - -**Do:** -- Create `src/lib/FilterBar.svelte` -- Props: `todos: Todo[]`, `filter: Filter`, `onFilterChange: (f: Filter) => void` -- Show count: "X items left" (incomplete count) -- Three filter buttons: All, Active, Completed -- Active filter is visually highlighted -- "Clear completed" button (hidden when no completed todos) -- Add component tests - -**Verify:** -- Tests pass -- Component renders count, filters, clear button - ---- - -### Task 8: App Integration - -Wire all components together in App.svelte. - -**Do:** -- Import all components and store -- Add filter state (default: 'all') -- Compute filtered todos based on filter state -- Render: heading, TodoInput, TodoList, FilterBar -- Pass appropriate props to each component - -**Verify:** -- App renders all components -- Adding todos works -- Toggling works -- Deleting works - ---- - -### Task 9: Filter Functionality - -Ensure filtering works end-to-end. - -**Do:** -- Verify filter buttons change displayed todos -- 'all' shows all todos -- 'active' shows only incomplete todos -- 'completed' shows only completed todos -- Clear completed removes completed todos and resets filter if needed -- Add integration tests - -**Verify:** -- Filter tests pass -- Manual verification of all filter states - ---- - -### Task 10: Styling and Polish - -Add CSS styling for usability. - -**Do:** -- Style the app to match the design mockup -- Completed todos have strikethrough and muted color -- Active filter button is highlighted -- Input has focus styles -- Delete button appears on hover (or always on mobile) -- Responsive layout - -**Verify:** -- App is visually usable -- Styles don't break functionality - ---- - -### Task 11: End-to-End Tests - -Add Playwright tests for full user flows. - -**Do:** -- Install Playwright: `npm init playwright@latest` -- Create `tests/todo.spec.ts` -- Test flows: - - Add a todo - - Complete a todo - - Delete a todo - - Filter todos - - Clear completed - - Persistence (add, reload, verify) - -**Verify:** -- `npx playwright test` passes - ---- - -### Task 12: README - -Document the project. - -**Do:** -- Create `README.md` with: - - Project description - - Setup: `npm install` - - Development: `npm run dev` - - Testing: `npm test` and `npx playwright test` - - Build: `npm run build` - -**Verify:** -- README accurately describes the project -- Instructions work diff --git a/tests/subagent-driven-dev/svelte-todo/scaffold.sh b/tests/subagent-driven-dev/svelte-todo/scaffold.sh deleted file mode 100755 index f7bef046aa..0000000000 --- a/tests/subagent-driven-dev/svelte-todo/scaffold.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# Scaffold the Svelte Todo test project -# Usage: ./scaffold.sh /path/to/target/directory - -set -e - -TARGET_DIR="${1:?Usage: $0 }" -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -# Create target directory -mkdir -p "$TARGET_DIR" -cd "$TARGET_DIR" - -# Initialize git repo -git init - -# Copy design and plan -cp "$SCRIPT_DIR/design.md" . -cp "$SCRIPT_DIR/plan.md" . - -# Create .claude settings to allow reads/writes in this directory -mkdir -p .claude -cat > .claude/settings.local.json << 'SETTINGS' -{ - "permissions": { - "allow": [ - "Read(**)", - "Edit(**)", - "Write(**)", - "Bash(npm:*)", - "Bash(npx:*)", - "Bash(mkdir:*)", - "Bash(git:*)" - ] - } -} -SETTINGS - -# Create initial commit -git add . -git commit -m "Initial project setup with design and plan" - -echo "Scaffolded Svelte Todo project at: $TARGET_DIR" -echo "" -echo "To run the test:" -echo " claude -p \"Execute this plan using superpowers:subagent-driven-development. Plan: $TARGET_DIR/plan.md\" --plugin-dir /path/to/superpowers"