-
Notifications
You must be signed in to change notification settings - Fork 0
613 lines (550 loc) · 31.5 KB
/
Copy pathci.yml
File metadata and controls
613 lines (550 loc) · 31.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
name: CI
on:
push:
branches: [main]
pull_request:
branches: [main]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
verify-plugin:
name: Verify plugin structure (${{ matrix.os }})
runs-on: ${{ matrix.os }}
timeout-minutes: 5
strategy:
fail-fast: false
matrix:
# T-12 (v1.7 audit Phase 3 Part C): macOS catches BSD/GNU userland
# divergence (mktemp template position, sed -i, find -exec) that
# ubuntu-only CI missed in PR #45. Windows deferred — bash-on-windows
# would require WSL action + script rewrites; tracked as follow-up.
os: [ubuntu-latest, macos-14]
steps:
- uses: actions/checkout@v7
- name: Set up Python 3.12
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Run verify-plugin.sh
run: bash scripts/verify-plugin.sh
- name: Validate manifests JSON
run: |
python3 -c "import json; json.load(open('.claude-plugin/marketplace.json'))"
python3 -c "import json; json.load(open('plugins/preview-forge/.claude-plugin/plugin.json'))"
python3 -c "import json; json.load(open('plugins/preview-forge/hooks/hooks.json'))"
python3 -c "import json; d=json.load(open('plugins/preview-forge/monitors/monitors.json')); assert isinstance(d, list), 'monitors.json must be top-level array'"
python3 -c "import json; json.load(open('plugins/preview-forge/settings.json'))"
for s in plugins/preview-forge/schemas/*.json; do python3 -c "import json; json.load(open('$s'))"; done
- name: Validate JSON Schemas
run: |
pip install jsonschema
python3 -c "
import json, jsonschema
for s in ['preview-card', 'panel-vote', 'score-report', 'pf-profile']:
schema = json.load(open(f'plugins/preview-forge/schemas/{s}.schema.json'))
jsonschema.Draft7Validator.check_schema(schema)
print(f'✓ {s}.schema.json is valid Draft-07')
"
- name: Validate all profiles against schema (v1.3+)
run: |
python3 -c "
import json, jsonschema
schema = json.load(open('plugins/preview-forge/schemas/pf-profile.schema.json'))
for name in ['standard', 'pro', 'max']:
p = json.load(open(f'plugins/preview-forge/profiles/{name}.json'))
jsonschema.validate(p, schema)
print(f'✓ {name} profile valid — {p[\"previews\"][\"count\"]} previews, {p[\"engineering\"][\"teams\"]}×{p[\"engineering\"][\"members_per_team\"]} eng, P95 {p[\"cost_ceiling\"][\"p95_tokens\"]:,} tok')
"
- name: Python hooks syntax
run: |
python3 -m py_compile plugins/preview-forge/hooks/factory-policy.py
python3 -m py_compile plugins/preview-forge/hooks/askuser-enforcement.py
python3 -m py_compile plugins/preview-forge/hooks/auto-retro-trigger.py
python3 -m py_compile plugins/preview-forge/hooks/idea-drift-detector.py
python3 -m py_compile plugins/preview-forge/hooks/cost-regression.py
python3 -m py_compile plugins/preview-forge/hooks/escalation-ledger.py
python3 -m py_compile scripts/standard-schema-lint.py
- name: Default profile is standard (v1.4+)
run: |
DEFAULT=$(python3 -c "import json; print(json.load(open('plugins/preview-forge/settings.json'))['pf']['defaultProfile'])")
[[ "$DEFAULT" == "standard" ]] || { echo "FAIL: defaultProfile expected 'standard', got '$DEFAULT'"; exit 1; }
echo "✓ settings.json defaultProfile = standard"
- name: Fixture suites (P1 security + P3 T-2/T-3/T-4/T-6/T-8/T-9.2 + P8 Q-9, v1.7 audit)
run: |
# jsonschema already installed a few steps up; the fixtures
# now fail-closed on missing dependency so this must stay in
# the same job that pip-installs it.
bash tests/fixtures/security/verify-security.sh
bash tests/fixtures/rule9-fp-guard/verify-rule9.sh
bash tests/fixtures/filled-ratio/verify-filled-ratio.sh
bash tests/fixtures/filled-ratio-gating/verify.sh
bash tests/fixtures/h1-modal-swap/verify.sh
bash tests/fixtures/normalize-constraints/verify-normalize.sh
bash tests/fixtures/lesson07-regression/verify-lesson07.sh
bash tests/fixtures/seed-expectations/verify-seed-expectations.sh
- name: T-7 e2e mock-bootstrap (3 profiles)
# Issue #79 (Option A): runs the deterministic /pf:new artifact
# pipeline against canned Socratic + Gate H1 responses. Catches
# regressions in: filled-ratio-gate, generate-gallery (iframe
# count), h1-modal-helper (browser/inline branches),
# lint-framework-convergence, generate-spec-anchor-audit.
# Pre-W3.9, clean-room manual runs were the only validation path —
# this step makes "demo day = first real run" no longer a failure
# mode. Same matrix as the rest of verify-plugin (ubuntu + macos)
# so BSD-vs-GNU userland divergence is also caught.
run: |
for profile in standard pro max; do
bash tests/e2e/mock-bootstrap.sh "$profile"
done
test-hooks:
name: Test hooks (unit)
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v7
- name: Set up Python 3.12
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Test factory-policy blocks destructive commands
env:
CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge
run: |
set +e
# safe — should exit 0
echo '{"tool_name":"Bash","tool_input":{"command":"ls -la"}}' | python3 plugins/preview-forge/hooks/factory-policy.py
[[ $? -eq 0 ]] || { echo "FAIL: safe command was blocked"; exit 1; }
# destructive — should exit 2
echo '{"tool_name":"Bash","tool_input":{"command":"docker push foo/bar"}}' | python3 plugins/preview-forge/hooks/factory-policy.py
[[ $? -eq 2 ]] || { echo "FAIL: docker push was not blocked"; exit 1; }
echo '{"tool_name":"Bash","tool_input":{"command":"rm -rf /"}}' | python3 plugins/preview-forge/hooks/factory-policy.py
[[ $? -eq 2 ]] || { echo "FAIL: rm -rf / was not blocked"; exit 1; }
echo '{"tool_name":"Bash","tool_input":{"command":"git push --force origin main"}}' | python3 plugins/preview-forge/hooks/factory-policy.py
[[ $? -eq 2 ]] || { echo "FAIL: force push to main was not blocked"; exit 1; }
# memory/ edit — should exit 2
echo '{"tool_name":"Edit","tool_input":{"file_path":"memory/LESSONS.md"}}' | python3 plugins/preview-forge/hooks/factory-policy.py
[[ $? -eq 2 ]] || { echo "FAIL: memory edit was not blocked"; exit 1; }
# auto-retro bypass — should exit 0
PF_AUTO_RETRO_BYPASS=1 bash -c "echo '{\"tool_name\":\"Edit\",\"tool_input\":{\"file_path\":\"memory/LESSONS.md\"}}' | python3 plugins/preview-forge/hooks/factory-policy.py"
[[ $? -eq 0 ]] || { echo "FAIL: auto-retro bypass did not work"; exit 1; }
echo "✓ factory-policy: 6/6 Rule 1-7 tests pass"
# Rule 8 — run artifact single-writer
# external writer tries chosen_preview.json → should exit 2
echo '{"tool_name":"Edit","tool_input":{"file_path":"/x/runs/r-001/chosen_preview.json"}}' | python3 plugins/preview-forge/hooks/factory-policy.py
[[ $? -eq 2 ]] || { echo "FAIL: external writer not blocked for chosen_preview.json"; exit 1; }
# M1 supervisor with PF_WRITER_ROLE=supervisor → should exit 0
PF_WRITER_ROLE=supervisor bash -c "echo '{\"tool_name\":\"Edit\",\"tool_input\":{\"file_path\":\"/x/runs/r-001/chosen_preview.json\"}}' | python3 plugins/preview-forge/hooks/factory-policy.py"
[[ $? -eq 0 ]] || { echo "FAIL: supervisor env bypass did not work"; exit 1; }
# external writer tries .frozen-hash → should exit 2
echo '{"tool_name":"Edit","tool_input":{"file_path":"/x/runs/r-001/.frozen-hash"}}' | python3 plugins/preview-forge/hooks/factory-policy.py
[[ $? -eq 2 ]] || { echo "FAIL: external writer not blocked for .frozen-hash"; exit 1; }
echo "✓ factory-policy Rule 8: 3/3 tests pass"
echo "✓ factory-policy: 9/9 total tests pass"
- name: Test askuser-enforcement warns on freeform
env:
CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge
run: |
set +e
OUTPUT=$(echo '{"tool_name":"Agent","subagent_type":"test","tool_response":{"output":"어떻게 하시겠어요?"}}' | python3 plugins/preview-forge/hooks/askuser-enforcement.py 2>&1)
[[ "$OUTPUT" == *"WARN"* ]] || { echo "FAIL: freeform question not warned"; exit 1; }
echo "✓ askuser-enforcement warns correctly"
- name: Test auto-retro-trigger enqueues Blackboard
run: |
cd /tmp && mkdir -p pf-ci-test && cd pf-ci-test
mkdir -p runs/r-001/score
OUTPUT=$(echo '{"tool_name":"Write","tool_input":{"file_path":"runs/r-001/score/report.json"}}' | CLAUDE_PLUGIN_ROOT=${{ github.workspace }}/plugins/preview-forge python3 ${{ github.workspace }}/plugins/preview-forge/hooks/auto-retro-trigger.py 2>&1)
[[ "$OUTPUT" == *"enqueued retro"* ]] || { echo "FAIL: auto-retro did not enqueue"; exit 1; }
[[ -f runs/r-001/blackboard.db ]] || { echo "FAIL: blackboard.db not created"; exit 1; }
echo "✓ auto-retro-trigger creates blackboard row"
- name: Test idea-drift-detector (v1.3+ Rule 9)
env:
CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge
run: |
python3 - <<'PYEOF'
import json, subprocess, os, tempfile, shutil, sys
tmp = tempfile.mkdtemp()
os.makedirs(f"{tmp}/runs/r-test/specs")
with open(f"{tmp}/runs/r-test/chosen_preview.json", "w") as f:
json.dump({
"advocate": "P10",
"title": "Minutes.ai API — meeting transcription as a service",
"idea_summary": "Developer-first REST API for asynchronous meeting transcription. Accept audio upload, return speaker-diarized transcript with timestamps. Priced per-minute. Target: SaaS companies adding meeting intelligence without building ML infra.",
"pitch": "One HTTP POST, webhook callback, transcript JSON out. No dashboard needed — just keys, webhooks, and the playground for debugging."
}, f)
HIGH = "# Minutes.ai API Product Spec\n\nRESTful endpoint for async meeting transcription. POST /v1/transcripts accepts audio file upload, returns job_id. Webhook fires with diarized transcript JSON when complete. Per-minute pricing with usage-based billing. Developer dashboard exposes API keys, webhook URLs, and a request playground for debugging REST integration. Target: SaaS companies adding meeting intelligence."
LOW = "# MeetingBot for Slack\n\nSlack bot that joins meeting channels, listens to voice calls, posts summaries as threaded replies. Install via OAuth. Team workspace billing model. Bot mentions trigger interactive summaries. Action buttons for follow-ups. Slack app directory listing. Notifications sent via Slack Events API. Installer onboarding. Workspace admin panel."
SHORT = "# Note\nShort edit."
env = os.environ.copy()
cases = [
("HIGH (on-idea)", {"tool_name": "Write", "tool_input": {"file_path": f"{tmp}/runs/r-test/specs/SPEC.md", "content": HIGH}}, env, 0),
("LOW (Slack pivot)", {"tool_name": "Write", "tool_input": {"file_path": f"{tmp}/runs/r-test/specs/SPEC.md", "content": LOW}}, env, 2),
("SHORT (<120)", {"tool_name": "Write", "tool_input": {"file_path": f"{tmp}/runs/r-test/specs/SPEC.md", "content": SHORT}}, env, 0),
("BYPASS", {"tool_name": "Write", "tool_input": {"file_path": f"{tmp}/runs/r-test/specs/SPEC.md", "content": LOW}}, {**env, "PF_DRIFT_BYPASS": "1"}, 0),
("non-protected", {"tool_name": "Write", "tool_input": {"file_path": f"{tmp}/notes.md", "content": LOW}}, env, 0),
]
failed = 0
for name, payload, e, expected in cases:
p = subprocess.run(["python3", "plugins/preview-forge/hooks/idea-drift-detector.py"], input=json.dumps(payload), env=e, capture_output=True, text=True)
if p.returncode == expected:
print(f" ✓ {name}: exit {p.returncode}")
else:
print(f" ✗ {name}: expected {expected}, got {p.returncode}")
failed += 1
shutil.rmtree(tmp)
if failed:
print(f"FAIL: {failed} drift-detector cases")
sys.exit(1)
print("✓ idea-drift-detector: 5/5 tests pass")
PYEOF
- name: Test cost-regression (v1.3+ P0-B)
env:
CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge
run: |
python3 - <<'PYEOF'
import json, subprocess, os, tempfile, shutil, sys
tmp = tempfile.mkdtemp()
env = os.environ.copy()
cases = [
("standard under P95", "standard", {"tokens_total": 30000, "elapsed_minutes": 10}, 0),
("standard P95 breach", "standard", {"tokens_total": 75000, "elapsed_minutes": 10}, 1),
("standard hard breach", "standard", {"tokens_total": 120000, "elapsed_minutes": 10}, 2),
("pro under P95", "pro", {"tokens_total": 100000, "elapsed_minutes": 30}, 0),
("pro hard breach (time)", "pro", {"tokens_total": 100000, "elapsed_minutes": 120}, 2),
("max hard breach", "max", {"tokens_total": 950000, "elapsed_minutes": 100}, 2),
]
failed = 0
for name, profile, snap, expected in cases:
run = f"{tmp}/runs/r-{profile}-{snap['tokens_total']}"
os.makedirs(run, exist_ok=True)
with open(f"{run}/.profile", "w") as f: f.write(profile)
with open(f"{run}/cost-snapshot.json", "w") as f: json.dump(snap, f)
p = subprocess.run(["python3", "plugins/preview-forge/hooks/cost-regression.py", run], env=env, capture_output=True, text=True)
if p.returncode == expected:
print(f" ✓ {name}: exit {p.returncode}")
else:
print(f" ✗ {name}: expected {expected}, got {p.returncode}")
failed += 1
# Canonical blackboard schema: confirm breach writes to `blackboard` (not `events`)
import sqlite3
breach_run = [r for name, profile, snap, expected in cases if expected > 0
for r in [f"{tmp}/runs/r-{profile}-{snap['tokens_total']}"]
if os.path.exists(f"{r}/blackboard.db")][0]
con = sqlite3.connect(f"{breach_run}/blackboard.db")
tables = [r[0] for r in con.execute("SELECT name FROM sqlite_master WHERE type='table'")]
if "blackboard" not in tables:
print(f" ✗ schema: canonical 'blackboard' table missing, found: {tables}")
failed += 1
else:
print(f" ✓ schema: writes to canonical 'blackboard' table (not 'events')")
# Defensive: malformed profile must NOT crash
bad_run = f"{tmp}/runs/r-malformed"
os.makedirs(bad_run, exist_ok=True)
with open(f"{bad_run}/.profile", "w") as f: f.write("nonexistent")
with open(f"{bad_run}/cost-snapshot.json", "w") as f: json.dump({"tokens_total": 999999}, f)
p = subprocess.run(["python3", "plugins/preview-forge/hooks/cost-regression.py", bad_run], env=env, capture_output=True, text=True)
if p.returncode == 0:
print(f" ✓ defensive: unknown profile returns 0 (no crash)")
else:
print(f" ✗ defensive: unknown profile returned {p.returncode}: {p.stderr}")
failed += 1
shutil.rmtree(tmp)
if failed:
print(f"FAIL: {failed} cost-regression cases")
sys.exit(1)
print("✓ cost-regression: 8/8 tests pass (6 classification + schema + defensive)")
PYEOF
- name: Test detect-surface (Proposal #2)
run: |
R1=$(bash scripts/detect-surface.sh <<<'{"text":"REST API for meeting transcription with webhooks."}')
echo "$R1" | grep -q '"surface": "rest-first"' || { echo "FAIL: REST case: $R1"; exit 1; }
R2=$(bash scripts/detect-surface.sh <<<'{"text":"Mobile dashboard with drag-drop layout and wizard onboarding."}')
echo "$R2" | grep -q '"surface": "ui-first"' || { echo "FAIL: UI case: $R2"; exit 1; }
R3=$(bash scripts/detect-surface.sh <<<'{"text":"Admin panel with dashboard UI and REST API for programmatic access. Self-service customer portal with settings page."}')
echo "$R3" | grep -q '"surface": "hybrid"' || { echo "FAIL: hybrid case: $R3"; exit 1; }
# Regression guard: grep -oc vs grep -o|wc -l — three "api" must score 3, not 1.
R4=$(bash scripts/detect-surface.sh <<<'{"text":"api api api"}')
echo "$R4" | grep -q '"rest": 3' || { echo "FAIL: grep occurrence count regression: $R4"; exit 1; }
# Security guard: command injection in idea text must not execute.
rm -f /tmp/pf-injection-canary
bash scripts/detect-surface.sh <<<'{"text":"`touch /tmp/pf-injection-canary` injected"}' > /dev/null
[[ ! -f /tmp/pf-injection-canary ]] || { echo "FAIL: command injection in idea text executed"; rm /tmp/pf-injection-canary; exit 1; }
echo "✓ detect-surface: 5/5 cases (3 classify + 1 occurrence + 1 security)"
- name: Test recommend-profile.sh (v1.4+ Phase P)
env:
CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge
run: |
set -e
# v1.4+ invocation: arg1=input path (/dev/stdin if piped), arg2=current profile
run() { bash scripts/recommend-profile.sh /dev/stdin "${2:-standard}" <<<"$1"; }
# Benign
r=$(run '{"text":"blog with markdown"}')
echo "$r" | grep -q '"action": "none"' || { echo "FAIL benign: $r"; exit 1; }
# Stripe → hard-require on standard (payments in standard's hard set)
r=$(run '{"text":"SaaS with Stripe subscription billing"}')
echo "$r" | grep -q '"action": "hard-require"' || { echo "FAIL stripe: $r"; exit 1; }
echo "$r" | grep -q '"payments"' || { echo "FAIL stripe signal: $r"; exit 1; }
# Profile-filter check: Stripe in pro context → NOT hard (payments NOT in pro's hard set)
r=$(run '{"text":"SaaS with Stripe subscription billing"}' pro)
echo "$r" | grep -q '"action": "none"' || { echo "FAIL stripe-in-pro: $r"; exit 1; }
# HIPAA in pro → hard (phi_healthcare IS in pro's hard set)
r=$(run '{"text":"EHR with HIPAA patient records"}' pro)
echo "$r" | grep -q '"action": "hard-require"' || { echo "FAIL hipaa-in-pro: $r"; exit 1; }
# Word-boundary: 'Delphi' should NOT match phi substring (Codex P2 fix)
r=$(run '{"text":"Delphi programming with morphism patterns"}')
echo "$r" | grep -q '"action": "none"' || { echo "FAIL delphi false-positive: $r"; exit 1; }
# PCI compliance without trailing space (Gemini fix)
r=$(run '{"text":"PCI compliance for card processing."}')
echo "$r" | grep -q '"action": "hard-require"' || { echo "FAIL pci-compliance: $r"; exit 1; }
# Korean
r=$(run '{"text":"기업용 멀티테넌트 SaaS, 감사로그와 결제 연동"}')
echo "$r" | grep -q '"action": "hard-require"' || { echo "FAIL korean: $r"; exit 1; }
# Multi-tenant + compliance (2 soft cats on standard, score 0.4 < 0.8 → hint)
r=$(run '{"text":"Multi-tenant workspace with SOC2 compliance"}')
echo "$r" | grep -q '"action": "hint"' || { echo "FAIL multi-tenant: $r"; exit 1; }
# Empty
r=$(run '{}')
echo "$r" | grep -q '"action": "none"' || { echo "FAIL empty: $r"; exit 1; }
# Injection canary
rm -f /tmp/pf-recommend-canary
run '{"text":"test `touch /tmp/pf-recommend-canary` end"}' > /dev/null
[[ ! -f /tmp/pf-recommend-canary ]] || { echo "FAIL: command injection executed"; rm /tmp/pf-recommend-canary; exit 1; }
echo "✓ recommend-profile: 10/10 cases pass (incl. profile-filter + word-boundary + injection canary)"
- name: Test escalation-ledger (v1.4+ Phase Q)
env:
PF_ESCALATION_LEDGER_DIR: /tmp/pf-ledger-test
run: |
set -e
rm -rf "$PF_ESCALATION_LEDGER_DIR"
LEDGER="python3 plugins/preview-forge/hooks/escalation-ledger.py"
# Empty lookup → exit 1
$LEDGER lookup abc123 2>/dev/null && { echo "FAIL: empty lookup returned 0"; exit 1; } || true
# Hash determinism + normalisation (case + dedup + sort)
H1=$($LEDGER hash "compliance,multi_tenant")
H2=$($LEDGER hash "multi_tenant,compliance")
H3=$($LEDGER hash "COMPLIANCE,compliance,Multi_Tenant")
[[ "$H1" == "$H2" ]] || { echo "FAIL: hash not sort-invariant"; exit 1; }
[[ "$H1" == "$H3" ]] || { echo "FAIL: hash not case/dedup-invariant"; exit 1; }
[[ ${#H1} -eq 64 ]] || { echo "FAIL: hash not full sha256 (got ${#H1} chars, expected 64)"; exit 1; }
# Empty replay_safe → 0 (safe)
$LEDGER replay_safe "$H1" || { echo "FAIL: empty replay_safe should exit 0"; exit 1; }
# Record decline
$LEDGER record "$H1" standard pro declined r-test-1 > /dev/null
# Now replay_safe suppresses
$LEDGER replay_safe "$H1" 2>/dev/null && { echo "FAIL: should suppress after decline"; exit 1; } || true
# Different signals not suppressed
H3=$($LEDGER hash "scale")
$LEDGER replay_safe "$H3" || { echo "FAIL: different signals wrongly suppressed"; exit 1; }
# Accepted decision not suppressed
$LEDGER record "$H1" standard pro accepted r-test-2 > /dev/null
$LEDGER replay_safe "$H1" || { echo "FAIL: accepted wrongly suppressed"; exit 1; }
# Concurrent write safety (fcntl lock — Gemini medium fix)
rm -rf "$PF_ESCALATION_LEDGER_DIR"
H_CONCURRENT=$($LEDGER hash "scale,enterprise_b2b")
for i in 1 2 3 4 5; do
$LEDGER record "$H_CONCURRENT" standard pro declined "r-concurrent-$i" > /dev/null &
done
wait
COUNT=$(python3 -c "import json; print(len(json.load(open('$PF_ESCALATION_LEDGER_DIR/escalation-history.json'))))")
[[ "$COUNT" -eq 5 ]] || { echo "FAIL: concurrent writes lost entries — got $COUNT expected 5"; exit 1; }
rm -rf "$PF_ESCALATION_LEDGER_DIR"
echo "✓ escalation-ledger: 9/9 cases pass (incl. 5-way concurrent write)"
- name: Test standard-schema-lint (v1.4+ Phase O)
run: |
set -e
cat > /tmp/portable.prisma <<'EOF'
datasource db { provider = "sqlite"; url = env("DATABASE_URL") }
model User {
id String @id @default(cuid())
role String
}
EOF
set +e
python3 scripts/standard-schema-lint.py /tmp/portable.prisma
CODE=$?
set -e
[[ "$CODE" -eq 0 ]] || { echo "FAIL portable: expected exit 0, got $CODE"; exit 1; }
cat > /tmp/unportable.prisma <<'EOF'
datasource db { provider = "sqlite" }
enum Role { A B }
model User { id String @id; role Role; data Json @db.JsonB }
EOF
set +e
python3 scripts/standard-schema-lint.py /tmp/unportable.prisma 2>/dev/null
CODE=$?
set -e
# Specifically assert exit 2 (non-portable detected), not just "non-zero" —
# so linter crashes, usage errors, import errors don't falsely pass (CodeRabbit minor).
[[ "$CODE" -eq 2 ]] || { echo "FAIL unportable: expected exit 2, got $CODE"; exit 1; }
rm /tmp/portable.prisma /tmp/unportable.prisma
echo "✓ schema-lint: 2/2 cases pass (exit codes asserted specifically)"
- name: Test preview-cache (Proposal #11)
env:
CLAUDE_PLUGIN_ROOT: ${{ github.workspace }}/plugins/preview-forge
PF_CACHE_DIR: /tmp/pf-cache-test
run: |
mkdir -p "$PF_CACHE_DIR"
K1=$(bash scripts/preview-cache.sh key "build todo app" pro)
K2=$(bash scripts/preview-cache.sh key "build todo app" pro)
[[ "$K1" == "$K2" ]] || { echo "FAIL: key not deterministic"; exit 1; }
K3=$(bash scripts/preview-cache.sh key "build todo app" standard)
[[ "$K1" != "$K3" ]] || { echo "FAIL: profile doesn't change key"; exit 1; }
# --previews=N override must produce a distinct key (same idea + profile, different N)
K4=$(bash scripts/preview-cache.sh key "build todo app" pro 9)
[[ "$K1" != "$K4" ]] || { echo "FAIL: --previews override didn't change key"; exit 1; }
echo '{"profile":"pro","previews":[]}' > /tmp/pf-test.json
bash scripts/preview-cache.sh put "$K1" /tmp/pf-test.json
bash scripts/preview-cache.sh get "$K1" > /dev/null || { echo "FAIL: get miss after put"; exit 1; }
# max profile should always miss
K_MAX=$(bash scripts/preview-cache.sh key "anything" max)
echo '{"profile":"max"}' > /tmp/pf-max.json
bash scripts/preview-cache.sh put "$K_MAX" /tmp/pf-max.json
bash scripts/preview-cache.sh get "$K_MAX" > /dev/null && { echo "FAIL: max shouldn't cache"; exit 1; } || true
rm -rf "$PF_CACHE_DIR" /tmp/pf-test.json /tmp/pf-max.json
echo "✓ preview-cache: 4/4 tests pass"
agent-counts:
name: Agent inventory (144 target)
runs-on: ubuntu-latest
timeout-minutes: 2
steps:
- uses: actions/checkout@v7
- name: Count agents
run: |
TOTAL=$(find plugins/preview-forge/agents -name "*.md" -type f | wc -l)
if [[ "$TOTAL" -ne 144 ]]; then
echo "::error::Expected 144 agents, found $TOTAL"
exit 1
fi
echo "✓ 144/144 agents present"
# per-dept (v1.5: scc 5 → 6 with scc-build-config)
declare -A EXPECTED=(
[meta]=3
[ideation]=29
[panels]=45
[spec]=9
[engineering]=25
[qa]=14
[scc]=6
[judges]=5
[auditors]=5
[docs]=3
)
for dept in "${!EXPECTED[@]}"; do
count=$(find plugins/preview-forge/agents/$dept -name "*.md" -type f | wc -l)
exp=${EXPECTED[$dept]}
if [[ "$count" -ne "$exp" ]]; then
echo "::error::$dept: $count (expected $exp)"
exit 1
fi
echo "✓ $dept: $count/$exp"
done
- name: All 26 advocates reference idea.spec.json (v1.7.0+ T-1)
run: |
COUNT=$(grep -l "idea\.spec\.json" plugins/preview-forge/agents/ideation/advocates/P*.md | wc -l | tr -d ' ')
if [[ "$COUNT" -ne 26 ]]; then
echo "::error::Expected 26 advocates referencing idea.spec.json, got $COUNT"
exit 1
fi
echo "✓ 26/26 advocates reference idea.spec.json"
- name: All 26 advocates share boilerplate (W2.6 / #61)
# Guards against silent boilerplate drift across the 26 P*.md
# advocate files — future schema-wide edits MUST hit every file.
run: bash tests/test-advocate-boilerplate.sh
- name: Check all agents use Opus 4.7
run: |
python3 <<'PYEOF'
import re, glob, sys
non_opus = []
for f in glob.glob("plugins/preview-forge/agents/**/*.md", recursive=True):
m = re.search(r'^model:\s*(.+)$', open(f).read(), re.MULTILINE)
if m and m.group(1).strip() not in ('opus', 'claude-opus-4-7', 'opus-4-7'):
non_opus.append((f, m.group(1).strip()))
if non_opus:
for f, m in non_opus:
print(f"::error file={f}::non-Opus model: {m}")
sys.exit(1)
print("✓ all 144 agents use Opus 4.7")
PYEOF
- name: Check agent name uniqueness
run: |
python3 <<'PYEOF'
import re, glob, sys
from collections import Counter
names = []
for f in glob.glob("plugins/preview-forge/agents/**/*.md", recursive=True):
m = re.search(r'^name:\s*(.+)$', open(f).read(), re.MULTILINE)
if m:
names.append(m.group(1).strip())
dups = {n: c for n, c in Counter(names).items() if c > 1}
if dups:
for n, c in dups.items():
print(f"::error::duplicate name: {n} ({c} times)")
sys.exit(1)
print(f"✓ all {len(names)} agent names are unique")
PYEOF
lint-shell:
name: Shell script lint
runs-on: ubuntu-latest
timeout-minutes: 2
steps:
- uses: actions/checkout@v7
- name: Install shellcheck
run: sudo apt-get update && sudo apt-get install -y shellcheck
- name: Lint bin/pf + verify-plugin.sh
run: |
shellcheck -x plugins/preview-forge/bin/pf || true
shellcheck -x scripts/verify-plugin.sh || true
shellcheck -x scripts/test-templates.sh || true
echo "✓ shellcheck complete (non-blocking)"
template-build:
name: Template build smoke (B1+B2 regression guard)
runs-on: ubuntu-latest
timeout-minutes: 6
steps:
- uses: actions/checkout@v7
- uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Static template content checks
run: bash scripts/test-templates.sh
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 9.15.0
- uses: actions/setup-node@v6
with:
node-version: 22
- name: Render templates to clean workspace
id: render
run: |
set -euo pipefail
WS="$RUNNER_TEMP/pf-template-smoke"
mkdir -p "$WS/lib"
for tpl in package.json tsconfig.json vitest.config.ts next.config.ts; do
cp "plugins/preview-forge/assets/${tpl}.standard.template" "$WS/$tpl"
done
# Substitute placeholders. (Since PR #13 the JSON templates are
# strict JSON — no `//` comment stripping needed; human guidance
# lives in package.json.standard.README.md.)
sed -i 's/{{PROJECT_NAME}}/pf-template-smoke/g' "$WS/package.json"
sed -i 's/{{NODE_VERSION}}/22/g' "$WS/package.json"
# Minimal typia-using source so `tsc --noEmit` exercises the transform plugin.
cat > "$WS/lib/check.ts" <<'TS'
import typia from "typia";
export interface Greeting { name: string; }
export const validate = typia.createValidate<Greeting>();
TS
touch "$WS/next-env.d.ts"
echo "ws=$WS" >> "$GITHUB_OUTPUT"
- name: pnpm install (smoke)
working-directory: ${{ steps.render.outputs.ws }}
run: pnpm install --no-frozen-lockfile
- name: pnpm typecheck (verifies typia AOT plugin wiring)
working-directory: ${{ steps.render.outputs.ws }}
run: pnpm typecheck
- name: Summary
if: always()
run: echo "✓ Templates pass static + install + typecheck. typia/vitest/unplugin chain intact."