Merge pull request #10 from DefangLabs/sync-3

Sync 3

Author: raphaeltm

.claude/rules/02-quality-gates.md

@@ -211,6 +211,7 @@ String containment tests on structured output create false confidence. The test
 3. **Existing workflows MUST be confirmed working.** Navigate the dashboard, projects, settings. Verify no regressions — pages load, data displays, navigation works, no new console errors.
 4. **New feature/fix MUST be verified on staging.** The specific changes in the PR must work correctly on the live staging environment.
 5. **Evidence MUST be reported.** Include screenshots, API responses, or Playwright observations in the PR.
+6. **For browser-consumed streams (SSE / WebSocket), verification MUST use a real browser, not `curl`.** `curl` can confirm bytes arrive on the wire; only a browser confirms the client actually dispatches them to its handler. See `.claude/rules/13-staging-verification.md` for the full reasoning and the post-mortem that motivated this rule (`docs/notes/2026-04-19-trial-sse-named-events-postmortem.md`).
 
 ### No Exceptions
 

.claude/rules/10-e2e-verification.md

@@ -23,6 +23,7 @@ Before marking a feature complete:
 - [ ] The test asserts the **final outcome**, not just that intermediate steps succeeded
 - [ ] If the test uses mocks at system boundaries, the mock asserts the **exact payload** the real system would receive
 - [ ] Any untestable gaps are documented with manual verification steps
+- [ ] **Port-of-pattern coverage** — when porting a multi-step pattern (VM boot, credential rotation, agent session lifecycle) from an existing consumer to a new one, the new consumer's tests MUST mock each cross-boundary target and assert **every step of the pattern fired** with the correct payload. A test that asserts "step 1 fired" but not "step 3 fired" does not prove the port is complete. See `docs/notes/2026-04-19-trial-orchestrator-agent-boot-postmortem.md` for the class of bug this prevents.
 
 ## Data Flow Tracing (Mandatory for Multi-Component Features)
 

.claude/rules/13-staging-verification.md

@@ -172,6 +172,13 @@ Match the verification to what the PR actually changes:
 - Verifying no console errors
 - "The code changes look correct"
 - "Unit tests pass"
+- **For browser-consumed streams (SSE / WebSocket): using `curl` to verify that
+  bytes arrive on the wire.** Curl confirms the *byte stream*; only a real
+  browser confirms *dispatch* to the client-side handler (`EventSource.onmessage`
+  or `WebSocket.onmessage`). A server can emit perfectly valid SSE that the
+  browser parses and then silently drops because nothing is listening for the
+  specific event name. See
+  `docs/notes/2026-04-19-trial-sse-named-events-postmortem.md`.
 
 These are baseline regression checks. They do NOT verify that the specific fix or feature works on the live environment.
 

.claude/skills/env-reference/SKILL.md

@@ -96,6 +96,21 @@ See `apps/api/.env.example` for the full list. Key variables:
 ### Credential Routes Rate Limits
 - `RATE_LIMIT_CREDENTIAL_UPDATE` — Applied to both user-scoped (`PUT /api/credentials/agent`) and project-scoped (`PUT /api/projects/:id/credentials`) credential write endpoints (MEDIUM #7 fix)
 
+### Trial Onboarding (`/try` flow)
+
+See `docs/guides/trial-configuration.md` for the full table with meanings and defaults. Summary:
+
+- `TRIAL_CLAIM_TOKEN_SECRET` — Worker secret; HMAC key for trial cookies (auto-provisioned by Pulumi)
+- `TRIAL_MONTHLY_CAP`, `TRIAL_WORKSPACE_TTL_MS`, `TRIAL_DATA_RETENTION_HOURS` — Global cap + lifetimes
+- `TRIAL_ANONYMOUS_USER_ID`, `TRIAL_ANONYMOUS_INSTALLATION_ID` — Sentinel rows for pre-claim ownership
+- `TRIAL_AGENT_TYPE_STAGING`, `TRIAL_AGENT_TYPE_PRODUCTION`, `TRIAL_DEFAULT_WORKSPACE_PROFILE` — Agent + profile selection
+- `TRIALS_ENABLED_KV_KEY`, `TRIAL_KILL_SWITCH_CACHE_MS` — Kill switch
+- `TRIAL_ORCHESTRATOR_OVERALL_TIMEOUT_MS`, `TRIAL_ORCHESTRATOR_STEP_MAX_RETRIES`, `TRIAL_ORCHESTRATOR_RETRY_BASE_DELAY_MS`, `TRIAL_ORCHESTRATOR_RETRY_MAX_DELAY_MS` — Orchestrator retry budget
+- `TRIAL_ORCHESTRATOR_NODE_READY_TIMEOUT_MS`, `TRIAL_ORCHESTRATOR_AGENT_READY_TIMEOUT_MS`, `TRIAL_ORCHESTRATOR_WORKSPACE_READY_TIMEOUT_MS`, `TRIAL_ORCHESTRATOR_WORKSPACE_READY_POLL_INTERVAL_MS` — Step-level timeouts
+- `TRIAL_VM_SIZE`, `TRIAL_VM_LOCATION` — VM overrides for trial workspaces
+- `TRIAL_GITHUB_TIMEOUT_MS` — Per-request timeout for the default-branch probe (`fetchDefaultBranch`); falls back to `main` on timeout/404/error
+- `TRIAL_KNOWLEDGE_GITHUB_TIMEOUT_MS`, `TRIAL_KNOWLEDGE_MAX_EVENTS` — Fast-path knowledge probe tunables
+
 ## VM Agent Environment Variables
 
 ### Container/User

.github/workflows/deploy-reusable.yml

@@ -404,6 +404,22 @@ jobs:
           PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
           CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
 
+      - name: Enable Trials on Staging (KV kill-switch)
+        # Trials are gated by a KV flag that fails closed. Staging always enables them
+        # automatically so the /try flow is live after every deploy. Production stays
+        # opt-in (operator flips the flag manually when ready to accept live traffic).
+        if: ${{ inputs.dry_run != true && inputs.environment == 'staging' }}
+        working-directory: apps/api
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
+        run: |
+          echo "Enabling trials kill-switch on staging (KV trials:enabled=true)"
+          # `wrangler kv key put` writes to remote by default; --remote is not a valid flag here.
+          pnpm exec wrangler kv key put "trials:enabled" "true" \
+            --binding KV \
+            --env staging
+
       - name: Configure Worker Secrets
         if: ${{ inputs.dry_run != true }}
         run: bash scripts/deploy/configure-secrets.sh ${{ inputs.environment }}
@@ -432,6 +448,7 @@ jobs:
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           SMOKE_TEST_AUTH_ENABLED: ${{ secrets.SMOKE_TEST_AUTH_ENABLED }}
+          ANTHROPIC_API_KEY_TRIAL: ${{ secrets.ANTHROPIC_API_KEY_TRIAL }}
           SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY }}
           GA4_API_SECRET: ${{ secrets.GA4_API_SECRET }}
           GA4_MEASUREMENT_ID: ${{ secrets.GA4_MEASUREMENT_ID }}

CLAUDE.md

@@ -221,6 +221,10 @@ Domains chain together: competitive research feeds marketing and business strate
 - Cloudflare D1 (credentials table with AES-GCM encrypted tokens) (028-provider-infrastructure)
 
 ## Recent Changes
+- ai-proxy-gateway: AI inference proxy routes LLM requests through Cloudflare AI Gateway — `POST /ai/v1/chat/completions` accepts OpenAI-format requests, transparently routes to Workers AI (@cf/* models) or Anthropic (claude-* models) with format translation (`ai-anthropic-translate.ts`); per-user RPM rate limiting + daily token budget via KV; admin model picker at `/admin/ai-proxy`; AI usage analytics dashboard at `/admin/analytics/ai-usage` aggregates AI Gateway logs by model, day, cost; configurable via AI_PROXY_ENABLED, AI_PROXY_DEFAULT_MODEL, AI_GATEWAY_ID, AI_PROXY_ALLOWED_MODELS, AI_PROXY_RATE_LIMIT_RPM, AI_PROXY_RATE_LIMIT_WINDOW_SECONDS, AI_PROXY_MAX_INPUT_TOKENS_PER_REQUEST, AI_USAGE_PAGE_SIZE, AI_USAGE_MAX_PAGES
+- trial-agent-boot: TrialOrchestrator `discovery_agent_start` step now runs the full 5-step idempotent VM boot (registers agent session via `createAgentSessionOnNode`, mints MCP token with trialId as synthetic taskId, `startAgentSessionOnNode` with discovery prompt + MCP server URL, drives ACP session `pending → assigned → running`; idempotency flags `mcpToken`, `agentSessionCreatedOnVm`, `agentStartedOnVm`, `acpAssignedOnVm`, `acpRunningOnVm` on DO state let crash/retry resume without double-booking); new `fetchDefaultBranch()` probes GitHub `/repos/:owner/:repo` with AbortController-bounded fetch and threads the real default branch through `projects.defaultBranch` + workspace `git clone --branch` (master-default repos like `octocat/Hello-World` now work); configurable via TRIAL_GITHUB_TIMEOUT_MS (default: 5000); new capability test `apps/api/tests/unit/durable-objects/trial-orchestrator-agent-boot.test.ts` asserts every cross-boundary call fires with correct payload; rule 10 updated with port-of-pattern coverage requirement. See `docs/notes/2026-04-19-trial-orchestrator-agent-boot-postmortem.md`.
+- trial-sse-events-fix: Fixed "zero trial.* events on staging" — `formatSse()` in `apps/api/src/routes/trial/events.ts` previously emitted named SSE frames (`event: trial.knowledge\ndata: {...}`), but the frontend subscribes via `source.onmessage` which only fires for the default (unnamed) event; frames arrived on the wire (curl saw them) but browser EventSource silently dropped them. Now emits unnamed `data: {JSON}\n\n` frames; the `TrialEvent` payload's own `type` discriminator preserves dispatch info. Also fixed `eventsUrl` in `apps/api/src/routes/trial/create.ts` response shape mismatch (`/api/trial/events?trialId=X` → `/api/trial/:trialId/events`). New capability test `apps/api/tests/workers/trial-event-bus-sse.test.ts` asserts no `event:` line + JSON round-trip across the TrialEventBus DO → SSE endpoint boundary; unit tests updated to assert new unnamed-frame contract and exact `eventsUrl` shape (no substring matches on URL contracts). Rule 13 updated to ban curl-only verification of browser-consumed SSE/WebSocket streams — curl confirms bytes, browsers confirm dispatch. See `docs/notes/2026-04-19-trial-sse-named-events-postmortem.md`.
+- trial-orchestrator-wire-up: TrialOrchestrator Durable Object + GitHub-API knowledge fast-path — `POST /api/trial/create` now fire-and-forget dispatches two concurrent `c.executionCtx.waitUntil` tasks: (1) `env.TRIAL_ORCHESTRATOR.idFromName(trialId)` DO state machine (alarm-driven, steps: project_creation → node_provisioning → workspace_creation → workspace_ready → agent_session → completed; idempotent `start()`; terminal guard on completed/failed; overall-timeout emits `trial.error`); (2) `emitGithubKnowledgeEvents()` probe hits unauthenticated `/repos/:o/:n`, `/repos/:o/:n/languages`, `/repos/:o/:n/readme` in parallel with AbortController-bounded fetches, emits up to `TRIAL_KNOWLEDGE_MAX_EVENTS` `trial.knowledge` events (description, primary language, stars, topics, license, language breakdown by bytes, README first paragraph), swallows all errors; `apps/api/src/services/trial/bridge.ts` bridges ACP session transitions (`running` → `trial.ready`, `failed` → `trial.error`) and MCP tool calls (`add_knowledge` → `trial.knowledge`, `create_idea` → `trial.idea`) into the SSE stream via `readTrialByProject()` KV lookup (no-op on non-trial projects); new sentinel `TRIAL_ANONYMOUS_INSTALLATION_ID` row in `github_installations` so trial projects satisfy the FK; configurable via TRIAL_ORCHESTRATOR_OVERALL_TIMEOUT_MS (default: 300000), TRIAL_ORCHESTRATOR_STEP_MAX_RETRIES (default: 5), TRIAL_ORCHESTRATOR_RETRY_BASE_DELAY_MS (default: 1000), TRIAL_ORCHESTRATOR_RETRY_MAX_DELAY_MS (default: 60000), TRIAL_ORCHESTRATOR_NODE_READY_TIMEOUT_MS (default: 180000), TRIAL_ORCHESTRATOR_AGENT_READY_TIMEOUT_MS (default: 60000), TRIAL_ORCHESTRATOR_WORKSPACE_READY_TIMEOUT_MS (default: 180000), TRIAL_ORCHESTRATOR_WORKSPACE_READY_POLL_INTERVAL_MS (default: 5000), TRIAL_VM_SIZE (default: DEFAULT_VM_SIZE), TRIAL_VM_LOCATION (default: DEFAULT_VM_LOCATION), TRIAL_KNOWLEDGE_GITHUB_TIMEOUT_MS (default: 5000), TRIAL_KNOWLEDGE_MAX_EVENTS (default: 10)
 - project-credential-overrides: Per-project agent credential overrides — `credentials.project_id` column (migration 0042, nullable FK to `projects.id ON DELETE CASCADE`) with two partial unique indexes (`WHERE project_id IS NULL` for user-scoped, `WHERE project_id IS NOT NULL` for project-scoped); `getDecryptedAgentKey(db, userId, agentType, key, projectId?)` resolves project → user → platform in order; workspace runtime callback forwards `workspace.projectId`; `CodexRefreshLock` DO preserves scope on OAuth token rotation; new `/api/projects/:id/credentials` routes (GET/PUT/DELETE) guarded by `requireOwnedProject` (404 on cross-user); `ProjectAgentsSection` on Project Settings combines credential override and model/permission override per agent using `AgentKeyCard` (scope='project') with inheritance hints; cross-user writes rejected at query layer AND ownership check; `autoActivate` only affects project-scoped rows (user-scoped untouched)
 - project-knowledge-graph: Per-project knowledge graph for persistent agent memory — `knowledge_entities`, `knowledge_observations`, `knowledge_relations` tables + FTS5 virtual table in ProjectData DO SQLite (migration 016); entity-observation-relation model with confidence scoring and recency weighting; 11 MCP tools (`add_knowledge`, `update_knowledge`, `remove_knowledge`, `get_knowledge`, `search_knowledge`, `get_project_knowledge`, `get_relevant_knowledge`, `relate_knowledge`, `get_related`, `confirm_knowledge`, `flag_contradiction`) in `apps/api/src/routes/mcp/knowledge-tools.ts`; auto-retrieval of relevant knowledge in `get_instructions` MCP tool; REST API at `/api/projects/:projectId/knowledge/*` for UI CRUD; Knowledge Browser page at `/projects/:id/knowledge` with entity list, search, type filters, detail panel; configurable via KNOWLEDGE_AUTO_RETRIEVE_LIMIT (default: 20), KNOWLEDGE_MAX_ENTITIES_PER_PROJECT (default: 500), KNOWLEDGE_MAX_OBSERVATIONS_PER_ENTITY (default: 100), KNOWLEDGE_SEARCH_LIMIT (default: 20), KNOWLEDGE_SEARCH_MAX_LIMIT (default: 100), KNOWLEDGE_LIST_PAGE_SIZE (default: 50), KNOWLEDGE_LIST_MAX_PAGE_SIZE (default: 200), KNOWLEDGE_OBSERVATION_MAX_LENGTH (default: 1000)
 - dispatch-task-config-parity: Full task execution config parity for `dispatch_task` MCP tool — extended schema accepts optional `agentProfileId`, `taskMode` (task/conversation), `agentType`, `workspaceProfile` (default/lightweight), `provider` (hetzner/scaleway/gcp), `vmLocation`; config precedence matches normal submit path: explicit field → agent profile → project default → platform default; `resolveAgentProfile()` from `agent-profiles.ts` resolves profiles by ID or name with built-in seeding; profile-derived values (`model`, `permissionMode`, `systemPromptAppend`) passed through to `startTaskRunnerDO()`; `agentProfileHint` and `taskMode` persisted in task INSERT for observability; location validated against resolved provider; `maxTurns`/`timeoutMinutes` excluded — not enforced by runtime (documented in task file)

apps/api/.env.example

@@ -46,6 +46,58 @@ BASE_DOMAIN=workspaces.example.com
 # MAX_SMOKE_TOKENS_PER_USER=10
 # MAX_SMOKE_TOKEN_NAME_LENGTH=100
 
+# ========================================
+# Trial Onboarding
+# ========================================
+# See docs/guides/trial-configuration.md for the full tunable list.
+# TRIAL_MONTHLY_CAP=1500
+# TRIAL_WORKSPACE_TTL_MS=1200000                # 20 min
+# TRIAL_DATA_RETENTION_HOURS=168                # 7 days
+# TRIAL_ANONYMOUS_USER_ID=system_anonymous_trials
+# TRIAL_ANONYMOUS_INSTALLATION_ID=1
+# TRIAL_AGENT_TYPE_STAGING=opencode
+# TRIAL_AGENT_TYPE_PRODUCTION=claude-code
+# TRIAL_DEFAULT_WORKSPACE_PROFILE=lightweight
+# TRIAL_VM_SIZE=cax11
+# TRIAL_VM_LOCATION=fsn1
+# TRIALS_ENABLED_KV_KEY=trials:enabled
+# TRIAL_REPO_MAX_KB=102400
+# TRIAL_COUNTER_KEEP_MONTHS=6
+# TRIAL_KILL_SWITCH_CACHE_MS=30000
+# TRIAL_GITHUB_TIMEOUT_MS=10000
+# TRIAL_WAITLIST_PURGE_DAYS=90
+# TRIAL_MODEL=@cf/qwen/qwen3-30b-a3b-fp8        # Model for trial conversations
+# TRIAL_LLM_PROVIDER=workers-ai                 # "workers-ai" or "anthropic"
+# ANTHROPIC_API_KEY_TRIAL=                       # Required only when TRIAL_LLM_PROVIDER=anthropic
+# ENVIRONMENT=staging                            # "staging" or "production" — set by deploy pipeline
+# TRIAL_SSE_HEARTBEAT_MS=15000
+# TRIAL_SSE_POLL_TIMEOUT_MS=15000
+# TRIAL_SSE_MAX_DURATION_MS=1800000
+# TRIAL_CRON_ROLLOVER_CRON="0 5 1 * *"
+# TRIAL_CRON_WAITLIST_CLEANUP="0 4 * * *"
+# TRIAL_KNOWLEDGE_GITHUB_TIMEOUT_MS=10000
+# TRIAL_KNOWLEDGE_MAX_EVENTS=50
+# TRIAL_ORCHESTRATOR_NODE_WAIT_MS=300000
+# TRIAL_ORCHESTRATOR_WORKSPACE_WAIT_MS=300000
+# TRIAL_ORCHESTRATOR_AGENT_BOOT_TIMEOUT_MS=120000
+# TRIAL_ORCHESTRATOR_STEP_RETRY_MAX=3
+# TRIAL_ORCHESTRATOR_STEP_RETRY_DELAY_MS=5000
+# RATE_LIMIT_TRIAL_CREATE=10
+# RATE_LIMIT_TRIAL_SSE=30
+
+# ========================================
+# AI Inference Proxy (Cloudflare AI Gateway)
+# ========================================
+# AI_PROXY_ENABLED=true
+# AI_PROXY_DEFAULT_MODEL=@cf/qwen/qwen3-30b-a3b-fp8
+# AI_GATEWAY_ID=sam
+# AI_PROXY_ALLOWED_MODELS=                       # Comma-separated; defaults in shared/constants
+# AI_PROXY_RATE_LIMIT_RPM=60
+# AI_PROXY_RATE_LIMIT_WINDOW_SECONDS=60
+# AI_PROXY_MAX_INPUT_TOKENS_PER_REQUEST=4096
+# AI_USAGE_PAGE_SIZE=50
+# AI_USAGE_MAX_PAGES=20
+
 # Pages project names (for Worker proxy routing)
 # PAGES_PROJECT_NAME=sam-web-prod
 # WWW_PAGES_PROJECT_NAME=sam-www
@@ -381,8 +433,8 @@ BASE_DOMAIN=workspaces.example.com
 
 # AI Inference Proxy (Workers AI gateway for trial/zero-config users)
 # AI_PROXY_ENABLED=true                                    # Kill switch: set "false" to disable (default: enabled)
-# AI_PROXY_DEFAULT_MODEL=@cf/meta/llama-4-scout-17b-16e-instruct  # Default Workers AI model
-# AI_PROXY_ALLOWED_MODELS=@cf/meta/llama-4-scout-17b-16e-instruct,@cf/qwen/qwen3-30b-a3b-fp8,@cf/google/gemma-3-12b-it
+# AI_PROXY_DEFAULT_MODEL=@cf/meta/llama-4-scout-17b-16e-instruct  # Default model (override via admin UI or env var)
+# AI_PROXY_ALLOWED_MODELS=@cf/meta/llama-4-scout-17b-16e-instruct,claude-haiku-4-5-20251001,@cf/qwen/qwen3-30b-a3b-fp8,@cf/google/gemma-3-12b-it
 # AI_PROXY_DAILY_INPUT_TOKEN_LIMIT=500000                  # Per-user daily input token cap
 # AI_PROXY_DAILY_OUTPUT_TOKEN_LIMIT=200000                 # Per-user daily output token cap
 # AI_PROXY_MAX_INPUT_TOKENS_PER_REQUEST=32000              # Max input tokens per single request

apps/api/package.json

@@ -18,23 +18,23 @@
     "deploy:staging": "wrangler deploy --env staging"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "3.1019.0",
-    "@aws-sdk/s3-request-presigner": "3.1019.0",
+    "@aws-sdk/client-s3": "3.1033.0",
+    "@aws-sdk/s3-request-presigner": "3.1033.0",
     "@hono/node-server": "1.19.9",
     "@hono/valibot-validator": "0.6.1",
     "@mastra/core": "1.9.0",
+    "@simple-agent-manager/cloud-init": "workspace:*",
     "@simple-agent-manager/providers": "workspace:*",
     "@simple-agent-manager/shared": "workspace:*",
-    "@simple-agent-manager/cloud-init": "workspace:*",
-    "ai": "6.0.156",
+    "ai": "6.0.168",
     "better-auth": "catalog:",
     "better-auth-cloudflare": "0.2.9",
     "drizzle-orm": "0.45.2",
-    "hono": "4.11.5",
+    "hono": "4.12.14",
     "jose": "6.2.2",
     "tsx": "catalog:",
     "valibot": "1.3.1",
-    "workers-ai-provider": "3.1.2"
+    "workers-ai-provider": "3.1.11"
   },
   "devDependencies": {
     "@cloudflare/vitest-pool-workers": "0.14.0",
@@ -45,7 +45,7 @@
     "drizzle-kit": "0.26.2",
     "eslint": "catalog:",
     "execa": "catalog:",
-    "fast-check": "4.5.3",
+    "fast-check": "4.7.0",
     "typescript": "catalog:",
     "vitest": "catalog:",
     "wrangler": "catalog:"