feat(v1.3.0): profiles (standard/pro/max) + Rule 9 drift + cost regression + 11 panel-validated changes #25
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| verify-plugin: | |
| name: Verify plugin structure | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Run verify-plugin.sh | |
| run: bash scripts/verify-plugin.sh | |
| - name: Validate manifests JSON | |
| run: | | |
| python3 -c "import json; json.load(open('.claude-plugin/marketplace.json'))" | |
| python3 -c "import json; json.load(open('plugins/preview-forge/.claude-plugin/plugin.json'))" | |
| python3 -c "import json; json.load(open('plugins/preview-forge/hooks/hooks.json'))" | |
| python3 -c "import json; d=json.load(open('plugins/preview-forge/monitors/monitors.json')); assert isinstance(d, list), 'monitors.json must be top-level array'" | |
| python3 -c "import json; json.load(open('plugins/preview-forge/settings.json'))" | |
| for s in plugins/preview-forge/schemas/*.json; do python3 -c "import json; json.load(open('$s'))"; done | |
| - name: Validate JSON Schemas | |
| run: | | |
| pip install jsonschema | |
| python3 -c " | |
| import json, jsonschema | |
| for s in ['preview-card', 'panel-vote', 'score-report', 'pf-profile']: | |
| schema = json.load(open(f'plugins/preview-forge/schemas/{s}.schema.json')) | |
| jsonschema.Draft7Validator.check_schema(schema) | |
| print(f'✓ {s}.schema.json is valid Draft-07') | |
| " | |
| - name: Validate all profiles against schema (v1.3+) | |
| run: | | |
| python3 -c " | |
| import json, jsonschema | |
| schema = json.load(open('plugins/preview-forge/schemas/pf-profile.schema.json')) | |
| for name in ['standard', 'pro', 'max']: | |
| p = json.load(open(f'plugins/preview-forge/profiles/{name}.json')) | |
| jsonschema.validate(p, schema) | |
| print(f'✓ {name} profile valid — {p[\"previews\"][\"count\"]} previews, {p[\"engineering\"][\"teams\"]}×{p[\"engineering\"][\"members_per_team\"]} eng, P95 {p[\"cost_ceiling\"][\"p95_tokens\"]:,} tok') | |
| " | |
| - name: Python hooks syntax | |
| run: | | |
| python3 -m py_compile plugins/preview-forge/hooks/factory-policy.py | |
| python3 -m py_compile plugins/preview-forge/hooks/askuser-enforcement.py | |
| python3 -m py_compile plugins/preview-forge/hooks/auto-retro-trigger.py | |
| python3 -m py_compile plugins/preview-forge/hooks/idea-drift-detector.py | |
| python3 -m py_compile plugins/preview-forge/hooks/cost-regression.py | |
| test-hooks: | |
| name: Test hooks (unit) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Test factory-policy blocks destructive commands | |
| env: | |
| CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge | |
| run: | | |
| set +e | |
| # safe — should exit 0 | |
| echo '{"tool_name":"Bash","tool_input":{"command":"ls -la"}}' | python3 plugins/preview-forge/hooks/factory-policy.py | |
| [[ $? -eq 0 ]] || { echo "FAIL: safe command was blocked"; exit 1; } | |
| # destructive — should exit 2 | |
| echo '{"tool_name":"Bash","tool_input":{"command":"docker push foo/bar"}}' | python3 plugins/preview-forge/hooks/factory-policy.py | |
| [[ $? -eq 2 ]] || { echo "FAIL: docker push was not blocked"; exit 1; } | |
| echo '{"tool_name":"Bash","tool_input":{"command":"rm -rf /"}}' | python3 plugins/preview-forge/hooks/factory-policy.py | |
| [[ $? -eq 2 ]] || { echo "FAIL: rm -rf / was not blocked"; exit 1; } | |
| echo '{"tool_name":"Bash","tool_input":{"command":"git push --force origin main"}}' | python3 plugins/preview-forge/hooks/factory-policy.py | |
| [[ $? -eq 2 ]] || { echo "FAIL: force push to main was not blocked"; exit 1; } | |
| # memory/ edit — should exit 2 | |
| echo '{"tool_name":"Edit","tool_input":{"file_path":"memory/LESSONS.md"}}' | python3 plugins/preview-forge/hooks/factory-policy.py | |
| [[ $? -eq 2 ]] || { echo "FAIL: memory edit was not blocked"; exit 1; } | |
| # auto-retro bypass — should exit 0 | |
| PF_AUTO_RETRO_BYPASS=1 bash -c "echo '{\"tool_name\":\"Edit\",\"tool_input\":{\"file_path\":\"memory/LESSONS.md\"}}' | python3 plugins/preview-forge/hooks/factory-policy.py" | |
| [[ $? -eq 0 ]] || { echo "FAIL: auto-retro bypass did not work"; exit 1; } | |
| echo "✓ factory-policy: 6/6 Rule 1-7 tests pass" | |
| # Rule 8 — run artifact single-writer | |
| # external writer tries chosen_preview.json → should exit 2 | |
| echo '{"tool_name":"Edit","tool_input":{"file_path":"/x/runs/r-001/chosen_preview.json"}}' | python3 plugins/preview-forge/hooks/factory-policy.py | |
| [[ $? -eq 2 ]] || { echo "FAIL: external writer not blocked for chosen_preview.json"; exit 1; } | |
| # M1 supervisor with PF_WRITER_ROLE=supervisor → should exit 0 | |
| PF_WRITER_ROLE=supervisor bash -c "echo '{\"tool_name\":\"Edit\",\"tool_input\":{\"file_path\":\"/x/runs/r-001/chosen_preview.json\"}}' | python3 plugins/preview-forge/hooks/factory-policy.py" | |
| [[ $? -eq 0 ]] || { echo "FAIL: supervisor env bypass did not work"; exit 1; } | |
| # external writer tries .frozen-hash → should exit 2 | |
| echo '{"tool_name":"Edit","tool_input":{"file_path":"/x/runs/r-001/.frozen-hash"}}' | python3 plugins/preview-forge/hooks/factory-policy.py | |
| [[ $? -eq 2 ]] || { echo "FAIL: external writer not blocked for .frozen-hash"; exit 1; } | |
| echo "✓ factory-policy Rule 8: 3/3 tests pass" | |
| echo "✓ factory-policy: 9/9 total tests pass" | |
| - name: Test askuser-enforcement warns on freeform | |
| env: | |
| CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge | |
| run: | | |
| set +e | |
| OUTPUT=$(echo '{"tool_name":"Agent","subagent_type":"test","tool_response":{"output":"어떻게 하시겠어요?"}}' | python3 plugins/preview-forge/hooks/askuser-enforcement.py 2>&1) | |
| [[ "$OUTPUT" == *"WARN"* ]] || { echo "FAIL: freeform question not warned"; exit 1; } | |
| echo "✓ askuser-enforcement warns correctly" | |
| - name: Test auto-retro-trigger enqueues Blackboard | |
| run: | | |
| cd /tmp && mkdir -p pf-ci-test && cd pf-ci-test | |
| mkdir -p runs/r-001/score | |
| OUTPUT=$(echo '{"tool_name":"Write","tool_input":{"file_path":"runs/r-001/score/report.json"}}' | CLAUDE_PLUGIN_ROOT=${{ github.workspace }}/plugins/preview-forge python3 ${{ github.workspace }}/plugins/preview-forge/hooks/auto-retro-trigger.py 2>&1) | |
| [[ "$OUTPUT" == *"enqueued retro"* ]] || { echo "FAIL: auto-retro did not enqueue"; exit 1; } | |
| [[ -f runs/r-001/blackboard.db ]] || { echo "FAIL: blackboard.db not created"; exit 1; } | |
| echo "✓ auto-retro-trigger creates blackboard row" | |
| - name: Test idea-drift-detector (v1.3+ Rule 9) | |
| env: | |
| CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge | |
| run: | | |
| python3 - <<'PYEOF' | |
| import json, subprocess, os, tempfile, shutil, sys | |
| tmp = tempfile.mkdtemp() | |
| os.makedirs(f"{tmp}/runs/r-test/specs") | |
| with open(f"{tmp}/runs/r-test/chosen_preview.json", "w") as f: | |
| json.dump({ | |
| "advocate": "P10", | |
| "title": "Minutes.ai API — meeting transcription as a service", | |
| "idea_summary": "Developer-first REST API for asynchronous meeting transcription. Accept audio upload, return speaker-diarized transcript with timestamps. Priced per-minute. Target: SaaS companies adding meeting intelligence without building ML infra.", | |
| "pitch": "One HTTP POST, webhook callback, transcript JSON out. No dashboard needed — just keys, webhooks, and the playground for debugging." | |
| }, f) | |
| HIGH = "# Minutes.ai API Product Spec\n\nRESTful endpoint for async meeting transcription. POST /v1/transcripts accepts audio file upload, returns job_id. Webhook fires with diarized transcript JSON when complete. Per-minute pricing with usage-based billing. Developer dashboard exposes API keys, webhook URLs, and a request playground for debugging REST integration. Target: SaaS companies adding meeting intelligence." | |
| LOW = "# MeetingBot for Slack\n\nSlack bot that joins meeting channels, listens to voice calls, posts summaries as threaded replies. Install via OAuth. Team workspace billing model. Bot mentions trigger interactive summaries. Action buttons for follow-ups. Slack app directory listing. Notifications sent via Slack Events API. Installer onboarding. Workspace admin panel." | |
| SHORT = "# Note\nShort edit." | |
| env = os.environ.copy() | |
| cases = [ | |
| ("HIGH (on-idea)", {"tool_name": "Write", "tool_input": {"file_path": f"{tmp}/runs/r-test/specs/SPEC.md", "content": HIGH}}, env, 0), | |
| ("LOW (Slack pivot)", {"tool_name": "Write", "tool_input": {"file_path": f"{tmp}/runs/r-test/specs/SPEC.md", "content": LOW}}, env, 2), | |
| ("SHORT (<120)", {"tool_name": "Write", "tool_input": {"file_path": f"{tmp}/runs/r-test/specs/SPEC.md", "content": SHORT}}, env, 0), | |
| ("BYPASS", {"tool_name": "Write", "tool_input": {"file_path": f"{tmp}/runs/r-test/specs/SPEC.md", "content": LOW}}, {**env, "PF_DRIFT_BYPASS": "1"}, 0), | |
| ("non-protected", {"tool_name": "Write", "tool_input": {"file_path": f"{tmp}/notes.md", "content": LOW}}, env, 0), | |
| ] | |
| failed = 0 | |
| for name, payload, e, expected in cases: | |
| p = subprocess.run(["python3", "plugins/preview-forge/hooks/idea-drift-detector.py"], input=json.dumps(payload), env=e, capture_output=True, text=True) | |
| if p.returncode == expected: | |
| print(f" ✓ {name}: exit {p.returncode}") | |
| else: | |
| print(f" ✗ {name}: expected {expected}, got {p.returncode}") | |
| failed += 1 | |
| shutil.rmtree(tmp) | |
| if failed: | |
| print(f"FAIL: {failed} drift-detector cases") | |
| sys.exit(1) | |
| print("✓ idea-drift-detector: 5/5 tests pass") | |
| PYEOF | |
| - name: Test cost-regression (v1.3+ P0-B) | |
| env: | |
| CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge | |
| run: | | |
| python3 - <<'PYEOF' | |
| import json, subprocess, os, tempfile, shutil, sys | |
| tmp = tempfile.mkdtemp() | |
| env = os.environ.copy() | |
| cases = [ | |
| ("standard under P95", "standard", {"tokens_total": 30000, "elapsed_minutes": 10}, 0), | |
| ("standard P95 breach", "standard", {"tokens_total": 75000, "elapsed_minutes": 10}, 1), | |
| ("standard hard breach", "standard", {"tokens_total": 120000, "elapsed_minutes": 10}, 2), | |
| ("pro under P95", "pro", {"tokens_total": 100000, "elapsed_minutes": 30}, 0), | |
| ("pro hard breach (time)", "pro", {"tokens_total": 100000, "elapsed_minutes": 120}, 2), | |
| ("max hard breach", "max", {"tokens_total": 950000, "elapsed_minutes": 100}, 2), | |
| ] | |
| failed = 0 | |
| for name, profile, snap, expected in cases: | |
| run = f"{tmp}/runs/r-{profile}-{snap['tokens_total']}" | |
| os.makedirs(run, exist_ok=True) | |
| with open(f"{run}/.profile", "w") as f: f.write(profile) | |
| with open(f"{run}/cost-snapshot.json", "w") as f: json.dump(snap, f) | |
| p = subprocess.run(["python3", "plugins/preview-forge/hooks/cost-regression.py", run], env=env, capture_output=True, text=True) | |
| if p.returncode == expected: | |
| print(f" ✓ {name}: exit {p.returncode}") | |
| else: | |
| print(f" ✗ {name}: expected {expected}, got {p.returncode}") | |
| failed += 1 | |
| # Canonical blackboard schema: confirm breach writes to `blackboard` (not `events`) | |
| import sqlite3 | |
| breach_run = [r for name, profile, snap, expected in cases if expected > 0 | |
| for r in [f"{tmp}/runs/r-{profile}-{snap['tokens_total']}"] | |
| if os.path.exists(f"{r}/blackboard.db")][0] | |
| con = sqlite3.connect(f"{breach_run}/blackboard.db") | |
| tables = [r[0] for r in con.execute("SELECT name FROM sqlite_master WHERE type='table'")] | |
| if "blackboard" not in tables: | |
| print(f" ✗ schema: canonical 'blackboard' table missing, found: {tables}") | |
| failed += 1 | |
| else: | |
| print(f" ✓ schema: writes to canonical 'blackboard' table (not 'events')") | |
| # Defensive: malformed profile must NOT crash | |
| bad_run = f"{tmp}/runs/r-malformed" | |
| os.makedirs(bad_run, exist_ok=True) | |
| with open(f"{bad_run}/.profile", "w") as f: f.write("nonexistent") | |
| with open(f"{bad_run}/cost-snapshot.json", "w") as f: json.dump({"tokens_total": 999999}, f) | |
| p = subprocess.run(["python3", "plugins/preview-forge/hooks/cost-regression.py", bad_run], env=env, capture_output=True, text=True) | |
| if p.returncode == 0: | |
| print(f" ✓ defensive: unknown profile returns 0 (no crash)") | |
| else: | |
| print(f" ✗ defensive: unknown profile returned {p.returncode}: {p.stderr}") | |
| failed += 1 | |
| shutil.rmtree(tmp) | |
| if failed: | |
| print(f"FAIL: {failed} cost-regression cases") | |
| sys.exit(1) | |
| print("✓ cost-regression: 8/8 tests pass (6 classification + schema + defensive)") | |
| PYEOF | |
| - name: Test detect-surface (Proposal #2) | |
| run: | | |
| R1=$(bash scripts/detect-surface.sh <<<'{"text":"REST API for meeting transcription with webhooks."}') | |
| echo "$R1" | grep -q '"surface": "rest-first"' || { echo "FAIL: REST case: $R1"; exit 1; } | |
| R2=$(bash scripts/detect-surface.sh <<<'{"text":"Mobile dashboard with drag-drop layout and wizard onboarding."}') | |
| echo "$R2" | grep -q '"surface": "ui-first"' || { echo "FAIL: UI case: $R2"; exit 1; } | |
| R3=$(bash scripts/detect-surface.sh <<<'{"text":"Admin panel with dashboard UI and REST API for programmatic access. Self-service customer portal with settings page."}') | |
| echo "$R3" | grep -q '"surface": "hybrid"' || { echo "FAIL: hybrid case: $R3"; exit 1; } | |
| # Regression guard: grep -oc vs grep -o|wc -l — three "api" must score 3, not 1. | |
| R4=$(bash scripts/detect-surface.sh <<<'{"text":"api api api"}') | |
| echo "$R4" | grep -q '"rest": 3' || { echo "FAIL: grep occurrence count regression: $R4"; exit 1; } | |
| # Security guard: command injection in idea text must not execute. | |
| rm -f /tmp/pf-injection-canary | |
| bash scripts/detect-surface.sh <<<'{"text":"`touch /tmp/pf-injection-canary` injected"}' > /dev/null | |
| [[ ! -f /tmp/pf-injection-canary ]] || { echo "FAIL: command injection in idea text executed"; rm /tmp/pf-injection-canary; exit 1; } | |
| echo "✓ detect-surface: 5/5 cases (3 classify + 1 occurrence + 1 security)" | |
| - name: Test preview-cache (Proposal #11) | |
| env: | |
| CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge | |
| PF_CACHE_DIR: /tmp/pf-cache-test | |
| run: | | |
| mkdir -p "$PF_CACHE_DIR" | |
| K1=$(bash scripts/preview-cache.sh key "build todo app" pro) | |
| K2=$(bash scripts/preview-cache.sh key "build todo app" pro) | |
| [[ "$K1" == "$K2" ]] || { echo "FAIL: key not deterministic"; exit 1; } | |
| K3=$(bash scripts/preview-cache.sh key "build todo app" standard) | |
| [[ "$K1" != "$K3" ]] || { echo "FAIL: profile doesn't change key"; exit 1; } | |
| # --previews=N override must produce a distinct key (same idea + profile, different N) | |
| K4=$(bash scripts/preview-cache.sh key "build todo app" pro 9) | |
| [[ "$K1" != "$K4" ]] || { echo "FAIL: --previews override didn't change key"; exit 1; } | |
| echo '{"profile":"pro","previews":[]}' > /tmp/pf-test.json | |
| bash scripts/preview-cache.sh put "$K1" /tmp/pf-test.json | |
| bash scripts/preview-cache.sh get "$K1" > /dev/null || { echo "FAIL: get miss after put"; exit 1; } | |
| # max profile should always miss | |
| K_MAX=$(bash scripts/preview-cache.sh key "anything" max) | |
| echo '{"profile":"max"}' > /tmp/pf-max.json | |
| bash scripts/preview-cache.sh put "$K_MAX" /tmp/pf-max.json | |
| bash scripts/preview-cache.sh get "$K_MAX" > /dev/null && { echo "FAIL: max shouldn't cache"; exit 1; } || true | |
| rm -rf "$PF_CACHE_DIR" /tmp/pf-test.json /tmp/pf-max.json | |
| echo "✓ preview-cache: 4/4 tests pass" | |
| agent-counts: | |
| name: Agent inventory (143 target) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Count agents | |
| run: | | |
| TOTAL=$(find plugins/preview-forge/agents -name "*.md" -type f | wc -l) | |
| if [[ "$TOTAL" -ne 143 ]]; then | |
| echo "::error::Expected 143 agents, found $TOTAL" | |
| exit 1 | |
| fi | |
| echo "✓ 143/143 agents present" | |
| # per-dept | |
| declare -A EXPECTED=( | |
| [meta]=3 | |
| [ideation]=29 | |
| [panels]=45 | |
| [spec]=9 | |
| [engineering]=25 | |
| [qa]=14 | |
| [scc]=5 | |
| [judges]=5 | |
| [auditors]=5 | |
| [docs]=3 | |
| ) | |
| for dept in "${!EXPECTED[@]}"; do | |
| count=$(find plugins/preview-forge/agents/$dept -name "*.md" -type f | wc -l) | |
| exp=${EXPECTED[$dept]} | |
| if [[ "$count" -ne "$exp" ]]; then | |
| echo "::error::$dept: $count (expected $exp)" | |
| exit 1 | |
| fi | |
| echo "✓ $dept: $count/$exp" | |
| done | |
| - name: Check all agents use Opus 4.7 | |
| run: | | |
| python3 <<'PYEOF' | |
| import re, glob, sys | |
| non_opus = [] | |
| for f in glob.glob("plugins/preview-forge/agents/**/*.md", recursive=True): | |
| m = re.search(r'^model:\s*(.+)$', open(f).read(), re.MULTILINE) | |
| if m and m.group(1).strip() not in ('opus', 'claude-opus-4-7', 'opus-4-7'): | |
| non_opus.append((f, m.group(1).strip())) | |
| if non_opus: | |
| for f, m in non_opus: | |
| print(f"::error file={f}::non-Opus model: {m}") | |
| sys.exit(1) | |
| print("✓ all 143 agents use Opus 4.7") | |
| PYEOF | |
| - name: Check agent name uniqueness | |
| run: | | |
| python3 <<'PYEOF' | |
| import re, glob, sys | |
| from collections import Counter | |
| names = [] | |
| for f in glob.glob("plugins/preview-forge/agents/**/*.md", recursive=True): | |
| m = re.search(r'^name:\s*(.+)$', open(f).read(), re.MULTILINE) | |
| if m: | |
| names.append(m.group(1).strip()) | |
| dups = {n: c for n, c in Counter(names).items() if c > 1} | |
| if dups: | |
| for n, c in dups.items(): | |
| print(f"::error::duplicate name: {n} ({c} times)") | |
| sys.exit(1) | |
| print(f"✓ all {len(names)} agent names are unique") | |
| PYEOF | |
| lint-shell: | |
| name: Shell script lint | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install shellcheck | |
| run: sudo apt-get update && sudo apt-get install -y shellcheck | |
| - name: Lint bin/pf + verify-plugin.sh | |
| run: | | |
| shellcheck -x plugins/preview-forge/bin/pf || true | |
| shellcheck -x scripts/verify-plugin.sh || true | |
| echo "✓ shellcheck complete (non-blocking)" |