diff --git a/.github/actions/nr-ci-event/action.yml b/.github/actions/nr-ci-event/action.yml new file mode 100644 index 0000000..4da401f --- /dev/null +++ b/.github/actions/nr-ci-event/action.yml @@ -0,0 +1,207 @@ +name: nr-ci-event +description: >- + Emit a CI test-run result to New Relic so a red CI run (test, e2e, smoke, + deploy gate) is studyable from NR dashboards, not just GitHub logs. Posts an + InstantCITestRun custom event on every gated job, plus an InstantCITestFailure + event when result=fail. No-ops cleanly (logs the payload it WOULD send) when + the NR secret/account is absent — never fails the calling job because NR is + unreachable (fork PRs, secret not yet provisioned). + +# Mechanism (CLAUDE.md design ref docs/ci/01-CI-INTEGRATION-DESIGN.md §NR +# observability): the NR Event API is a single HTTP POST to +# https://insights-collector.newrelic.com/v1/accounts//events +# authenticated with the ingest license key (the SAME NEW_RELIC_LICENSE_KEY the +# Go agents use at runtime — it is a valid Insert Key for the Event API). The +# account id is the numeric NEW_RELIC_ACCOUNT_ID. Both are passed as inputs from +# repo secrets by the caller. When EITHER is empty the action prints the payload +# and exits 0 (the no-op-without-secret contract — rule: never red a PR because +# NR is down). EU-region accounts override the collector host via nr-region. + +inputs: + # --- NR credentials (caller passes from secrets; empty => no-op) --- + license-key: + description: >- + NR ingest license key (Insert Key for the Event API). Pass + `secrets.NEW_RELIC_LICENSE_KEY`. Empty => action no-ops (dry-run log). + required: false + default: '' + account-id: + description: >- + Numeric NR account id. Pass `secrets.NEW_RELIC_ACCOUNT_ID`. + Empty => action no-ops (dry-run log). + required: false + default: '' + nr-region: + description: 'US (default) or EU — selects the insights-collector host.' + required: false + default: 'US' + + # --- event payload (caller fills from the GitHub context + job result) --- + result: + description: 'pass | fail — usually `job.status == ''success'' && ''pass'' || ''fail''`.' + required: true + suite: + description: >- + Logical suite name, e.g. build-and-test, coverage, playwright, pr-smoke, + e2e-prod, deploy-gate. The dashboard FACETs on this. + required: true + # NOTE: composite actions cannot read the `github` context in their own + # expressions, so the caller MUST pass these from its `with:` block (e.g. + # repo: `github.repository`). The defaults below only apply when a caller + # omits them entirely. + repo: + description: 'Repository (owner/name). Caller passes `github.repository`.' + required: false + default: '' + workflow: + description: 'Workflow name. Caller passes `github.workflow`.' + required: false + default: '' + branch: + description: 'Branch ref name. Caller passes `github.ref_name`.' + required: false + default: '' + commit-sha: + description: 'Commit SHA under test. Caller passes `github.sha`.' + required: false + default: '' + pr-number: + description: 'PR number (empty on push). Caller passes `github.event.pull_request.number`.' + required: false + default: '' + duration-ms: + description: 'Suite duration in milliseconds (0 if not measured).' + required: false + default: '0' + failed-step: + description: 'On failure: the step/phase that failed (free text, no PII). Empty on pass.' + required: false + default: '' + log-url: + description: 'URL to the run logs for triage. Caller passes the run URL.' + required: false + default: '' + event-name: + description: 'GitHub event name. Caller passes `github.event_name`.' + required: false + default: '' + actor: + description: 'GitHub actor. Caller passes `github.actor`.' + required: false + default: '' + +runs: + using: composite + steps: + - name: Emit CI result to New Relic (no-op without secret) + shell: bash + env: + # Composite actions may reference ONLY `inputs` (+ env/runner/steps) in + # their expressions — `secrets`, `job`, and `github` are NOT available + # here, so the CALLER resolves those and passes them as inputs. All + # untrusted/free-form values flow through env, never interpolated into + # the shell body (injection-safe — same posture as ci.yml's + # dispatch-auth-contract-e2e job). + NR_LICENSE_KEY: ${{ inputs.license-key }} + NR_ACCOUNT_ID: ${{ inputs.account-id }} + NR_REGION: ${{ inputs.nr-region }} + EV_RESULT: ${{ inputs.result }} + EV_SUITE: ${{ inputs.suite }} + EV_REPO: ${{ inputs.repo }} + EV_WORKFLOW: ${{ inputs.workflow }} + EV_BRANCH: ${{ inputs.branch }} + EV_COMMIT: ${{ inputs.commit-sha }} + EV_PR: ${{ inputs.pr-number }} + EV_DURATION_MS: ${{ inputs.duration-ms }} + EV_FAILED_STEP: ${{ inputs.failed-step }} + EV_LOG_URL: ${{ inputs.log-url }} + EV_EVENT_NAME: ${{ inputs.event-name }} + EV_ACTOR: ${{ inputs.actor }} + run: | + set -uo pipefail + + # Normalise the result to the pass|fail enum the dashboard FACETs on. + # Anything that isn't exactly "pass" is treated as "fail" so a typo or + # a cancelled job reads as a non-pass (conservative — never a false green). + case "${EV_RESULT}" in + pass) RESULT="pass" ;; + *) RESULT="fail" ;; + esac + + # Build the InstantCITestRun event (always) and, on fail, the + # InstantCITestFailure event. jq composes the JSON so every value is + # passed as an argument (no shell concatenation of free-form text). + DURATION="${EV_DURATION_MS}" + case "${DURATION}" in ''|*[!0-9]*) DURATION=0 ;; esac + + RUN_EVENT=$(jq -n -c \ + --arg eventType "InstantCITestRun" \ + --arg repo "${EV_REPO}" \ + --arg workflow "${EV_WORKFLOW}" \ + --arg branch "${EV_BRANCH}" \ + --arg commit_sha "${EV_COMMIT}" \ + --arg pr_number "${EV_PR}" \ + --arg result "${RESULT}" \ + --arg suite "${EV_SUITE}" \ + --arg event_name "${EV_EVENT_NAME}" \ + --arg actor "${EV_ACTOR}" \ + --arg log_url "${EV_LOG_URL}" \ + --argjson duration_ms "${DURATION}" \ + '{eventType:$eventType, repo:$repo, workflow:$workflow, branch:$branch, + commit_sha:$commit_sha, pr_number:$pr_number, result:$result, + suite:$suite, event_name:$event_name, actor:$actor, log_url:$log_url, + duration_ms:$duration_ms}') + + PAYLOAD="[${RUN_EVENT}]" + if [ "${RESULT}" = "fail" ]; then + FAIL_EVENT=$(jq -n -c \ + --arg eventType "InstantCITestFailure" \ + --arg repo "${EV_REPO}" \ + --arg workflow "${EV_WORKFLOW}" \ + --arg branch "${EV_BRANCH}" \ + --arg commit_sha "${EV_COMMIT}" \ + --arg pr_number "${EV_PR}" \ + --arg suite "${EV_SUITE}" \ + --arg failed_step "${EV_FAILED_STEP}" \ + --arg log_url "${EV_LOG_URL}" \ + --arg event_name "${EV_EVENT_NAME}" \ + '{eventType:$eventType, repo:$repo, workflow:$workflow, branch:$branch, + commit_sha:$commit_sha, pr_number:$pr_number, suite:$suite, + failed_step:$failed_step, log_url:$log_url, event_name:$event_name}') + PAYLOAD="[${RUN_EVENT},${FAIL_EVENT}]" + fi + + # No-op-without-secret contract: print what WOULD be sent and exit 0 so + # a fork PR (no secret) or an unprovisioned repo never reds because NR + # is unreachable. + if [ -z "${NR_LICENSE_KEY}" ] || [ -z "${NR_ACCOUNT_ID}" ]; then + echo "::notice title=nr-ci-event::NEW_RELIC_LICENSE_KEY or NEW_RELIC_ACCOUNT_ID absent — dry-run only (no event sent)." + echo "would POST to NR Event API the following payload:" + echo "${PAYLOAD}" | jq . + exit 0 + fi + + case "$(echo "${NR_REGION}" | tr '[:lower:]' '[:upper:]')" in + EU) HOST="insights-collector.eu01.nr-data.net" ;; + *) HOST="insights-collector.newrelic.com" ;; + esac + URL="https://${HOST}/v1/accounts/${NR_ACCOUNT_ID}/events" + + echo "POSTing ${RESULT} result for suite='${EV_SUITE}' to NR account ${NR_ACCOUNT_ID} (${HOST})" + HTTP_CODE=$(curl -sS -o /tmp/nr_ci_event.out -w '%{http_code}' \ + -X POST "${URL}" \ + -H "Content-Type: application/json" \ + -H "Api-Key: ${NR_LICENSE_KEY}" \ + --data-binary "${PAYLOAD}" || echo "000") + + echo "NR Event API responded HTTP ${HTTP_CODE}" + cat /tmp/nr_ci_event.out 2>/dev/null || true + echo + + # NR returns 200 on accept. Any other code (incl. network failure 000) + # is logged as a warning but NEVER fails the job — observability must + # not gate the pipeline. + if [ "${HTTP_CODE}" != "200" ]; then + echo "::warning title=nr-ci-event::NR Event API returned ${HTTP_CODE} (expected 200). CI result not recorded in NR; not failing the job." + fi + exit 0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbc1946..dee89dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,28 @@ jobs: INSTANT_API_REPO: ${{ github.workspace }}/../api run: go test ./... -v -race -count=1 + # Wave 5 — push the gated-test result to New Relic so a red run is + # studyable from an NR dashboard, not just the GitHub Actions log. + # if: always() records a FAILED `go test` as InstantCITestRun result=fail + # + InstantCITestFailure. No-ops cleanly without the NR secret. + - name: Emit CI result to New Relic + if: always() + uses: ./.github/actions/nr-ci-event + with: + license-key: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + account-id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} + result: ${{ job.status == 'success' && 'pass' || 'fail' }} + suite: build-and-test + pr-number: ${{ github.event.pull_request.number }} + failed-step: ${{ job.status != 'success' && 'go build / vet / test (-race)' || '' }} + repo: ${{ github.repository }} + workflow: ${{ github.workflow }} + branch: ${{ github.ref_name }} + commit-sha: ${{ github.sha }} + log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + event-name: ${{ github.event_name }} + actor: ${{ github.actor }} + # ------------------------------------------------------------------ # GATING integration job (W2-T1). # @@ -212,3 +234,24 @@ jobs: env: INSTANT_API_REPO: ${{ github.workspace }}/../api run: go test ./internal/... -run Integration -count=1 -p 1 -v + + # Wave 5 — record the real-DB integration-gate outcome in NR + # (suite=integration) so a red here is visible alongside the unit gate on + # the CI-health dashboard. No-ops without the NR secret. + - name: Emit integration result to New Relic + if: always() + uses: ./.github/actions/nr-ci-event + with: + license-key: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + account-id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} + result: ${{ job.status == 'success' && 'pass' || 'fail' }} + suite: integration + pr-number: ${{ github.event.pull_request.number }} + failed-step: ${{ job.status != 'success' && 'real-DB integration tests' || '' }} + repo: ${{ github.repository }} + workflow: ${{ github.workflow }} + branch: ${{ github.ref_name }} + commit-sha: ${{ github.sha }} + log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + event-name: ${{ github.event_name }} + actor: ${{ github.actor }} diff --git a/internal/jobs/flow_synthetic.go b/internal/jobs/flow_synthetic.go index 9b6141f..6543361 100644 --- a/internal/jobs/flow_synthetic.go +++ b/internal/jobs/flow_synthetic.go @@ -375,7 +375,28 @@ func (w *FlowSyntheticWorker) Work(ctx context.Context, job *river.Job[FlowSynth // flows (recorded per-flow below). seedOK := w.ensureSyntheticTeam(ctx) - results := w.runMatrix(ctx, runID, commitID, seedOK) + // Mint the session JWT once for every authed leg (P0 + money). If the seed + // failed OR the mint fails, the authed legs degrade (config/DB drift, not an + // outage) — recorded per-flow inside each matrix. + bearer := "" + if seedOK { + var err error + bearer, err = w.mintSessionJWT() + if err != nil { + seedOK = false + slog.Warn("jobs.flow_synthetic.mint_failed", "reason", err.Error()) + } + } + + results := w.runMatrix(ctx, runID, commitID, bearer, seedOK) + + // Money/value-journey legs (claim, deploy gate, checkout, magic-link → + // Brevo classification) — the value paths a paying customer's money rides + // on, each reading a truth surface (rule 12). Same flag, same counters/event, + // same matrix. See flow_synthetic_money.go. + for flow, res := range w.runMoneyMatrix(ctx, runID, commitID, bearer, seedOK) { + results[flow] = res + } // Reaper backstop: sweep any synthetic resource an earlier mid-leg crash // left behind. Inline reap already handles the happy path; this catches @@ -408,7 +429,7 @@ type flowSyntheticResult struct { // runMatrix executes every (enabled) flow in the P0 set sequentially and // returns a flow→result map for the completion log. Each flow is wrapped by // runFlow (timeout + recover + record). -func (w *FlowSyntheticWorker) runMatrix(ctx context.Context, runID, commitID string, seedOK bool) map[string]string { +func (w *FlowSyntheticWorker) runMatrix(ctx context.Context, runID, commitID, bearer string, seedOK bool) map[string]string { out := map[string]string{} // Flow 1 — healthz (anonymous, no auth). @@ -416,18 +437,9 @@ func (w *FlowSyntheticWorker) runMatrix(ctx context.Context, runID, commitID str return w.flowHealthz(fctx) }) - // Flows 2 & 3 require the seeded team + a minted session JWT. If the seed - // failed they degrade (config/DB drift, not an outage) so they don't page. - bearer := "" - if seedOK { - var err error - bearer, err = w.mintSessionJWT() - if err != nil { - seedOK = false - slog.Warn("jobs.flow_synthetic.mint_failed", "reason", err.Error()) - } - } - + // Flows 2 & 3 require the seeded team + the minted session JWT (minted once + // in Work and passed in). If the seed failed / mint failed seedOK is false + // and they degrade (config/DB drift, not an outage) so they don't page. out[flowAuthMe] = w.runFlow(ctx, runID, commitID, flowAuthMe, func(fctx context.Context) flowSyntheticResult { if !seedOK { return flowSyntheticResult{flow: flowAuthMe, actor: flowActorHuman, tier: w.cfg.Tier, result: flowResultDegraded, reason: "synthetic team/JWT unavailable — flow skipped"} @@ -489,6 +501,14 @@ func flowActorForFlow(flow string) string { return flowActorHuman case flowProvisionReap: return flowActorAgent + case flowClaim: + return flowActorAnon + case flowMagicLink: + return flowActorHuman + case flowCheckout: + return flowActorHuman + case flowDeployStatus: + return flowActorAgent default: return analyticsevent.ActorUnknown } diff --git a/internal/jobs/flow_synthetic_money.go b/internal/jobs/flow_synthetic_money.go new file mode 100644 index 0000000..2540647 --- /dev/null +++ b/internal/jobs/flow_synthetic_money.go @@ -0,0 +1,501 @@ +package jobs + +// flow_synthetic_money.go — the money/value-journey legs of the continuous +// synthetic flow matrix (Wave 5, docs/ci/01-CI-INTEGRATION-DESIGN.md §NR +// observability + docs/sessions/2026-06-04/TEST-ACCOUNTS-AND-NR-SYNTHETICS-PLAN.md). +// +// flow_synthetic.go runs the P0 read/provision legs (healthz, auth_me, +// provision_reap). This file adds the legs that watch the *value* paths — the +// flows a paying customer's money rides on — each reading a TRUTH SURFACE +// (CLAUDE.md rule 12), never a proxy: +// +// - flow_claim (human, api): GET /claim/preview with a deliberately +// invalid token, asserting the claim endpoint is up AND enforcing its +// single-use/JWT contract (400 invalid_token). Read-only — the claim path +// is single-use-consuming, so a synthetic claim can't fire a real /claim +// POST every tick without burning onboarding rows; the preview surface is +// the safe, side-effect-free contract probe. Truth surface: the rendered +// contract response, not a 200 health ping. +// +// - flow_deploy_status (agent, api): POST /deploy/new on the synthetic free +// team, asserting the TIER-GATE contract (402 + agent_action) rather than +// building a real app every 5 min (a real build per tick would churn +// postgres-customers + Kaniko — the design defers real deploy_new to a +// staging lane). Truth surface: the deploy gate's 402 wall (the exact +// surface a free user hits) — proves the deploy entitlement gate is live. +// When the synthetic tier has deploy headroom (hobby+), a 201/202 is also +// accepted and the created app is reaped via the deploy delete path. +// +// - flow_checkout (human, api): POST /api/v1/billing/checkout with the +// synthetic session, asserting the endpoint is REACHABLE and does not 5xx +// (contract-only — Razorpay live recurring is operator-blocked, so a real +// charge can't be driven; a 402/409/502-with-known-body is an acceptable +// "blocked-but-alive" shape, a 500 crash is a fail). Truth surface: the +// checkout endpoint's non-crash response — catches a checkout handler that +// panics/regresses even while Razorpay itself is blocked. +// +// - flow_magic_link (human, api+db): POST /auth/email/start (202), then +// read the forwarder_sent ledger's terminal `classification` for the +// synthetic recipient (rule 12: the ledger row is the truth surface for +// "did the email actually go out", NOT Brevo's 201/our 202). The result's +// reason carries the classification (delivered / rejected / bounced_* / +// deferred / …) so the matrix dashboard surfaces the REAL email-delivery +// health — today that means the Brevo-sender-unvalidated reality +// (project_brevo_sender_not_validated.md) shows as classification=rejected +// rather than a false green from the 202. +// +// All four are flag-gated by the SAME master FLOW_SYNTHETIC_ENABLED flag and +// honour the per-flow FLOW_SYNTHETIC_DISABLED kill list, so a flapping money +// leg can be silenced without killing the P0 suite. They emit the same +// instant_flow_test_* counters + InstantFlowTest event (cohort=synthetic) as +// the P0 legs, so one dashboard renders the whole grid. + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// Money/value-journey flow ids — the `flow` label + AttrFlow attribute. As with +// the P0 flow ids these are constants (rule 16: one source per contract token) +// and a registry test asserts the full set. +const ( + flowClaim = "claim" + flowDeployStatus = "deploy_status" + flowCheckout = "checkout" + flowMagicLink = "magic_link" +) + +// flowSyntheticInvalidClaimToken is the deliberately-malformed token the claim +// preview leg sends. It is NOT a real JWT, so ClaimPreview returns 400 +// invalid_token — the safe, single-use-preserving contract probe. A constant so +// the assertion + the value stay in one place. +const flowSyntheticInvalidClaimToken = "synthetic-invalid-claim-token-not-a-jwt" + +// Per-flow latency budgets for the money legs. Crossing the budget records +// result=degraded (slow-but-correct) rather than failing — same posture as the +// P0 legs in flowSyntheticLegLatencyBudgets. magic_link gets the widest budget: +// /auth/email/start does a rate-limit hash + a Redis write before returning 202. +var flowSyntheticMoneyLatencyBudgets = map[string]time.Duration{ + flowClaim: 2 * time.Second, + flowDeployStatus: 5 * time.Second, + flowCheckout: 5 * time.Second, + flowMagicLink: 3 * time.Second, +} + +// runMoneyMatrix executes the money/value-journey legs sequentially, mirroring +// runMatrix's per-flow isolation (runFlow handles timeout + recover + record). +// Called from Work after the P0 matrix. The anon claim leg needs no auth; the +// other three need the seeded team + a minted session JWT, so they degrade (not +// page) when seedOK is false (config/DB drift, not an outage). +func (w *FlowSyntheticWorker) runMoneyMatrix(ctx context.Context, runID, commitID, bearer string, seedOK bool) map[string]string { + out := map[string]string{} + + // claim — anonymous read-only contract probe (no auth needed). + out[flowClaim] = w.runFlow(ctx, runID, commitID, flowClaim, func(fctx context.Context) flowSyntheticResult { + return w.flowClaimPreview(fctx) + }) + + // magic_link — anon POST + a DB truth-surface read; needs the DB (for the + // forwarder_sent read) but not the session JWT. Degrades when db is nil. + out[flowMagicLink] = w.runFlow(ctx, runID, commitID, flowMagicLink, func(fctx context.Context) flowSyntheticResult { + return w.flowMagicLink(fctx) + }) + + // deploy_status + checkout need the authenticated session. + out[flowDeployStatus] = w.runFlow(ctx, runID, commitID, flowDeployStatus, func(fctx context.Context) flowSyntheticResult { + if !seedOK { + return flowSyntheticResult{flow: flowDeployStatus, actor: flowActorAgent, tier: w.cfg.Tier, result: flowResultDegraded, reason: "synthetic team/JWT unavailable — flow skipped"} + } + return w.flowDeployStatus(fctx, runID, bearer) + }) + + out[flowCheckout] = w.runFlow(ctx, runID, commitID, flowCheckout, func(fctx context.Context) flowSyntheticResult { + if !seedOK { + return flowSyntheticResult{flow: flowCheckout, actor: flowActorHuman, tier: w.cfg.Tier, result: flowResultDegraded, reason: "synthetic team/JWT unavailable — flow skipped"} + } + return w.flowCheckout(fctx, bearer) + }) + + return out +} + +// moneyBudgetFor returns the per-flow latency budget for a money leg, honouring +// a test override (SetBudgetOverrideForTest) the same way budgetFor does for the +// P0 legs. +func (w *FlowSyntheticWorker) moneyBudgetFor(flow string) time.Duration { + if w.budgetOverride != nil { + if d, ok := w.budgetOverride[flow]; ok { + return d + } + } + return flowSyntheticMoneyLatencyBudgets[flow] +} + +// ─── claim ────────────────────────────────────────────────────────────────── + +// flowClaimPreview drives GET /claim/preview?t= and asserts the claim +// endpoint enforces its JWT contract by returning 400 invalid_token. This is the +// read-only, single-use-preserving truth surface for "is the claim path alive +// and validating tokens" — a 200/302/5xx instead of the 400 contract means the +// claim funnel (anon→claimed, the first money step) is broken. +func (w *FlowSyntheticWorker) flowClaimPreview(ctx context.Context) flowSyntheticResult { + budget := w.moneyBudgetFor(flowClaim) + r := flowSyntheticResult{flow: flowClaim, actor: flowActorAnon, tier: "anonymous"} + + target := w.cfg.BaseURL + "/claim/preview?t=" + url.QueryEscape(flowSyntheticInvalidClaimToken) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + r.result = flowResultFail + r.reason = "build_request: " + err.Error() + return r + } + req.Header.Set("User-Agent", "instanode-flow-synthetic/1") + + start := time.Now() + resp, err := w.httpCli.Do(req) + r.latency = time.Since(start) + if err != nil { + r.result = flowResultFail + r.reason = "http_error: " + err.Error() + return r + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + + r.observeLatency = true + r.httpStatus = resp.StatusCode + // The contract: a bad token → 400 with error "invalid_token" (or + // "missing_token" if the query is dropped). Any 2xx/3xx/5xx is a contract + // breach: the claim path is not validating, or it crashed. + if resp.StatusCode != http.StatusBadRequest { + r.result = flowResultFail + r.reason = fmt.Sprintf("status=%d (want 400 invalid_token contract); body=%s", resp.StatusCode, truncateForLog(string(body), 256)) + return r + } + var parsed struct { + Error string `json:"error"` + } + _ = json.Unmarshal(body, &parsed) + if parsed.Error != "invalid_token" && parsed.Error != "missing_token" { + r.result = flowResultFail + r.reason = fmt.Sprintf("400 but unexpected error=%q (want invalid_token); body=%s", parsed.Error, truncateForLog(string(body), 256)) + return r + } + if r.latency > budget { + r.result = flowResultDegraded + r.reason = fmt.Sprintf("latency=%dms over budget=%dms", r.latency.Milliseconds(), budget.Milliseconds()) + return r + } + r.result = flowResultPass + return r +} + +// ─── deploy_status ──────────────────────────────────────────────────────────── + +// flowDeployStatus drives POST /deploy/new on the synthetic team and asserts the +// tier-entitlement gate. For a free/anonymous tier (no deploy headroom) the +// contract is a 402 wall — the exact surface a free user hits — which proves the +// deploy gate is live WITHOUT building a real app every tick. When the synthetic +// tier has headroom (hobby+), a 201/202 is accepted and the created app is reaped +// via the deploy delete path (cleanup ledger, rule 24). +func (w *FlowSyntheticWorker) flowDeployStatus(ctx context.Context, runID, bearer string) flowSyntheticResult { + budget := w.moneyBudgetFor(flowDeployStatus) + r := flowSyntheticResult{flow: flowDeployStatus, actor: flowActorAgent, tier: w.cfg.Tier} + + // A minimal multipart-free probe: POST an empty JSON body. The tier gate is + // evaluated before the tarball is parsed for the no-headroom tiers, so a + // free-tier probe reaches the 402 wall on body-shape alone. (For headroom + // tiers the handler would 400 on the missing tarball; we treat a 400 here as + // "gate passed, build inputs missing" which is still a non-5xx contract.) + target := w.cfg.BaseURL + "/deploy/new" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader("{}")) + if err != nil { + r.result = flowResultFail + r.reason = "build_request: " + err.Error() + return r + } + req.Header.Set("Authorization", "Bearer "+bearer) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "instanode-flow-synthetic/1") + + start := time.Now() + resp, err := w.httpCli.Do(req) + r.latency = time.Since(start) + if err != nil { + r.result = flowResultFail + r.reason = "http_error: " + err.Error() + return r + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + + r.observeLatency = true + r.httpStatus = resp.StatusCode + + switch { + case resp.StatusCode == http.StatusPaymentRequired: + // The free/anon tier-gate wall — the expected truth surface for a + // no-headroom synthetic team. Gate is live → pass. + if r.latency > budget { + r.result = flowResultDegraded + r.reason = fmt.Sprintf("402 gate ok but latency=%dms over budget=%dms", r.latency.Milliseconds(), budget.Milliseconds()) + return r + } + r.result = flowResultPass + return r + case resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusAccepted: + // Headroom tier accepted the deploy — reap the created app so we never + // leak (rule 24). Best-effort id extraction. + var parsed struct { + ID string `json:"id"` + } + _ = json.Unmarshal(body, &parsed) + if parsed.ID != "" { + w.reapDeployment(ctx, bearer, parsed.ID, runID) + } + r.result = flowResultPass + return r + case resp.StatusCode >= 400 && resp.StatusCode < 500: + // A 4xx other than 402 (e.g. 400 missing-tarball on a headroom tier, or + // 413 over-cap) means the gate/validation responded — non-5xx contract + // held. Record as pass with the status in the reason for visibility. + r.result = flowResultPass + r.reason = fmt.Sprintf("deploy gate responded %d (non-5xx contract held)", resp.StatusCode) + return r + default: + r.result = flowResultFail + r.reason = fmt.Sprintf("status=%d (deploy endpoint 5xx/crash); body=%s", resp.StatusCode, truncateForLog(string(body), 256)) + return r + } +} + +// reapDeployment deletes a synthetic deployment via the real DELETE +// /api/v1/deployments/:id path (cleanup ledger, rule 24). Best-effort: a leaked +// outcome is recorded on instant_flow_synthetic_reaped_total{flow,outcome} so a +// failed reap is visible. Mirrors reapResource. +func (w *FlowSyntheticWorker) reapDeployment(ctx context.Context, bearer, deployID, runID string) { + target := w.cfg.BaseURL + "/api/v1/deployments/" + url.PathEscape(deployID) + req, _ := http.NewRequestWithContext(ctx, http.MethodDelete, target, nil) + req.Header.Set("Authorization", "Bearer "+bearer) + req.Header.Set("User-Agent", "instanode-flow-synthetic/1") + + resp, err := w.httpCli.Do(req) + if err != nil { + w.recordReap(ctx, flowDeployStatus, reapOutcomeLeaked, deployID, runID, "http_error: "+err.Error()) + return + } + defer func() { _ = resp.Body.Close() }() + _, _ = io.ReadAll(io.LimitReader(resp.Body, 1024)) + if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusAccepted || resp.StatusCode == http.StatusNotFound { + w.recordReap(ctx, flowDeployStatus, reapOutcomeReaped, deployID, runID, "") + return + } + w.recordReap(ctx, flowDeployStatus, reapOutcomeLeaked, deployID, runID, fmt.Sprintf("delete status=%d", resp.StatusCode)) +} + +// ─── checkout ───────────────────────────────────────────────────────────────── + +// flowCheckout drives POST /api/v1/billing/checkout with the synthetic session +// and asserts the endpoint is reachable + does not 5xx. Razorpay live recurring +// is operator-blocked (project_razorpay_recurring_not_enabled.md), so a real +// charge can't be driven — a 402/409/502-with-known-body (Razorpay-blocked) is +// an acceptable "alive-but-blocked" shape. Only a 5xx CRASH (a checkout handler +// panic/regression) fails the leg. Truth surface: the checkout endpoint's +// non-crash response. +func (w *FlowSyntheticWorker) flowCheckout(ctx context.Context, bearer string) flowSyntheticResult { + budget := w.moneyBudgetFor(flowCheckout) + r := flowSyntheticResult{flow: flowCheckout, actor: flowActorHuman, tier: w.cfg.Tier} + + // Probe an upgrade to a paid tier. The body shape mirrors the dashboard's + // "Upgrade to Pro" call; the handler validates the plan before touching + // Razorpay, so even a blocked account exercises the handler path. + target := w.cfg.BaseURL + "/api/v1/billing/checkout" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader(`{"plan":"pro"}`)) + if err != nil { + r.result = flowResultFail + r.reason = "build_request: " + err.Error() + return r + } + req.Header.Set("Authorization", "Bearer "+bearer) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "instanode-flow-synthetic/1") + + start := time.Now() + resp, err := w.httpCli.Do(req) + r.latency = time.Since(start) + if err != nil { + r.result = flowResultFail + r.reason = "http_error: " + err.Error() + return r + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + + r.observeLatency = true + r.httpStatus = resp.StatusCode + // Contract-only: any non-5xx means the checkout handler is alive and + // responded (success URL, or a known blocked/validation status). A 5xx is a + // handler crash/regression → fail. + if resp.StatusCode >= 500 { + r.result = flowResultFail + r.reason = fmt.Sprintf("status=%d (checkout endpoint 5xx/crash); body=%s", resp.StatusCode, truncateForLog(string(body), 256)) + return r + } + if r.latency > budget { + r.result = flowResultDegraded + r.reason = fmt.Sprintf("checkout responded %d but latency=%dms over budget=%dms", resp.StatusCode, r.latency.Milliseconds(), budget.Milliseconds()) + return r + } + r.result = flowResultPass + r.reason = fmt.Sprintf("checkout endpoint alive (status=%d; contract-only, Razorpay live operator-blocked)", resp.StatusCode) + return r +} + +// ─── magic_link ─────────────────────────────────────────────────────────────── + +// flowMagicLink drives POST /auth/email/start (expect 202) then reads the +// forwarder_sent ledger's terminal classification for the most recent synthetic +// magic-link send. The 202 only proves the request was accepted; the LEDGER ROW +// is the truth surface (rule 12) for whether the email actually went out. The +// result's reason carries the classification so the dashboard surfaces the REAL +// delivery health (today: classification=rejected because the Brevo sender is +// unvalidated — project_brevo_sender_not_validated.md). A non-202 from +// /auth/email/start, OR a forwarder_sent row with a terminal failure +// classification, fails the leg; a 'success'/'delivered' classification (or no +// row yet — the forwarder is async) passes. +func (w *FlowSyntheticWorker) flowMagicLink(ctx context.Context) flowSyntheticResult { + budget := w.moneyBudgetFor(flowMagicLink) + r := flowSyntheticResult{flow: flowMagicLink, actor: flowActorHuman, tier: w.cfg.Tier} + + target := w.cfg.BaseURL + "/auth/email/start" + bodyJSON := fmt.Sprintf(`{"email":%q}`, w.cfg.Email) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader(bodyJSON)) + if err != nil { + r.result = flowResultFail + r.reason = "build_request: " + err.Error() + return r + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "instanode-flow-synthetic/1") + + start := time.Now() + resp, err := w.httpCli.Do(req) + r.latency = time.Since(start) + if err != nil { + r.result = flowResultFail + r.reason = "http_error: " + err.Error() + return r + } + defer func() { _ = resp.Body.Close() }() + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + + r.observeLatency = true + r.httpStatus = resp.StatusCode + // /auth/email/start always returns 202 (even for unknown emails — it never + // leaks account existence) or 400 for a malformed body. A non-202 means the + // send-request path is broken. + if resp.StatusCode != http.StatusAccepted { + r.result = flowResultFail + r.reason = fmt.Sprintf("status=%d (want 202 from /auth/email/start); body=%s", resp.StatusCode, truncateForLog(string(respBody), 256)) + return r + } + + // Truth surface: read the most-recent forwarder_sent classification. The + // forwarder is async (the worker dispatches on its own tick), so a freshly + // requested send may not have a row yet — that is NOT a failure (the request + // was accepted). We read the latest row to surface delivery health. + classification, ok := w.latestForwarderClassification(ctx) + if !ok { + // No ledger row readable (db nil, or no send recorded yet) — the 202 + // path held; report the request-accepted state without a delivery verdict. + if r.latency > budget { + r.result = flowResultDegraded + r.reason = fmt.Sprintf("202 accepted; no forwarder_sent row yet (async); latency=%dms over budget=%dms", r.latency.Milliseconds(), budget.Milliseconds()) + return r + } + r.result = flowResultPass + r.reason = "202 accepted; no forwarder_sent classification readable yet (forwarder is async)" + return r + } + + // A terminal-failure classification is the real, rule-12 truth that email + // is NOT reaching inboxes (today: rejected, the unvalidated-sender reality). + if isForwarderFailureClassification(classification) { + r.result = flowResultFail + r.reason = fmt.Sprintf("magic-link email classification=%q (truth surface: email not delivered)", classification) + return r + } + if r.latency > budget { + r.result = flowResultDegraded + r.reason = fmt.Sprintf("classification=%q but latency=%dms over budget=%dms", classification, r.latency.Milliseconds(), budget.Milliseconds()) + return r + } + r.result = flowResultPass + r.reason = fmt.Sprintf("magic-link email classification=%q", classification) + return r +} + +// forwarderFailureClassifications enumerates the forwarder_sent.classification +// terminal-failure values (the rule-12 "email did NOT reach the inbox" set). +// 'success'/'delivered' (and the async no-row case) are the healthy outcomes. +// A var (not const) so a test can read the set; not mutated at runtime. +var forwarderFailureClassifications = map[string]bool{ + "rejected": true, // Brevo accepted (201) then internally rejected — the unvalidated-sender reality + "bounced_hard": true, + "bounced_soft": true, + "complaint": true, + "error": true, + "permanent_drop": true, + "transient": true, +} + +// isForwarderFailureClassification reports whether a forwarder_sent +// classification is a terminal delivery failure. Case-insensitive + trimmed so a +// stored-value casing drift can't mask a failure as healthy. +func isForwarderFailureClassification(c string) bool { + return forwarderFailureClassifications[strings.ToLower(strings.TrimSpace(c))] +} + +// IsForwarderFailureClassificationForTest is an exported test seam over the +// unexported classifier so the external _test package can assert the +// healthy-vs-failure partition without a DB round-trip. +func IsForwarderFailureClassificationForTest(c string) bool { + return isForwarderFailureClassification(c) +} + +// latestForwarderClassification reads the most-recent forwarder_sent row's +// terminal classification for the synthetic recipient, the rule-12 truth surface +// for magic-link delivery. Returns (classification, true) on a readable row, or +// ("", false) when db is nil OR no row exists yet (the forwarder is async, so a +// just-requested send legitimately has no row — the caller treats that as +// "request accepted, no verdict yet", not a failure). recipient is stored +// masked, so the match is on template_kind (magic-link kinds) ordered by sent_at. +func (w *FlowSyntheticWorker) latestForwarderClassification(ctx context.Context) (string, bool) { + if w.db == nil { + return "", false + } + var classification string + // template_kind LIKE '%magic%' OR '%login%' — the magic-link send kinds. + // LIMIT 1 over sent_at DESC = the latest synthetic-relevant send. A short + // window (1h) keeps a stale row from masking a current outage. + err := w.db.QueryRowContext(ctx, ` + SELECT classification + FROM forwarder_sent + WHERE (template_kind ILIKE '%magic%' OR template_kind ILIKE '%login%' OR template_kind ILIKE '%link%') + AND sent_at > now() - interval '1 hour' + ORDER BY sent_at DESC + LIMIT 1 + `).Scan(&classification) + if err != nil { + // sql.ErrNoRows (no recent send) OR a query error → no verdict. Both + // degrade to "no row" (the caller passes the 202-accepted state). + return "", false + } + return classification, true +} diff --git a/internal/jobs/flow_synthetic_money_test.go b/internal/jobs/flow_synthetic_money_test.go new file mode 100644 index 0000000..e153ed9 --- /dev/null +++ b/internal/jobs/flow_synthetic_money_test.go @@ -0,0 +1,405 @@ +package jobs_test + +// flow_synthetic_money_test.go — branch-coverage tests for the money/value +// journey legs (flow_synthetic_money.go): claim, deploy_status, checkout, and +// magic_link. The shared harness (newFlowAPIServer / newFixture / happyFlowState +// / expectForwarder*) lives in flow_synthetic_test.go; these tests drive the +// per-leg contract + error + degraded branches the happy-path sweep doesn't hit +// (headroom-tier deploy + reap, checkout 5xx, magic_link failure classification, +// the no-row-async path, build_request failures, the classifier test seam). + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + + "instant.dev/common/analyticsevent" + "instant.dev/worker/internal/jobs" +) + +// TestFlowMoney_DeployHeadroom_AcceptedThenReaped drives the headroom-tier +// branch of flowDeployStatus: /deploy/new returns 201 with an id → the leg +// reaps the created app via DELETE /api/v1/deployments/:id (cleanup ledger, +// rule 24) and passes. Covers reapDeployment's happy path. +func TestFlowMoney_DeployHeadroom_AcceptedThenReaped(t *testing.T) { + st := happyFlowState() + st.deployStatus = http.StatusCreated + st.deployBody = `{"id":"dep-11111111-1111-4111-8111-111111111111"}` + st.deployDelStat = http.StatusOK + srv := newFlowAPIServer(st) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + expectReapAudit(f.mock) // provision_reap inline reap + expectForwarderClassification(f.mock, "success") // magic_link + expectReapAudit(f.mock) // deploy_status reap ledger row + expectOrphanSweepEmpty(f.mock) + + f.run(t) + + if got := f.fm.resultFor("deploy_status"); got != analyticsevent.ResultPass { + t.Errorf("deploy headroom: want pass, got %q", got) + } + var deployReaped bool + for _, r := range f.fm.reapOutcomes() { + if r.flow == "deploy_status" && r.outcome == "reaped" { + deployReaped = true + } + } + if !deployReaped { + t.Error("deploy headroom: want a deploy_status reaped outcome (cleanup ledger)") + } +} + +// TestFlowMoney_DeployReapLeak drives reapDeployment's leak branch: the deploy +// is accepted (201) but the DELETE returns 500 → the app leaked (recorded on +// the reap counter). The leg itself still passes (the gate/create contract held; +// the leak is surfaced separately, same posture as the provision reap). +func TestFlowMoney_DeployReapLeak(t *testing.T) { + st := happyFlowState() + st.deployStatus = http.StatusCreated + st.deployBody = `{"id":"dep-22222222-2222-4222-8222-222222222222"}` + st.deployDelStat = http.StatusInternalServerError // delete fails → leak + srv := newFlowAPIServer(st) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + expectReapAudit(f.mock) // provision_reap inline reap + expectForwarderClassification(f.mock, "success") // magic_link + expectReapAudit(f.mock) // deploy_status leaked ledger row + expectOrphanSweepEmpty(f.mock) + + f.run(t) + + var deployLeaked bool + for _, r := range f.fm.reapOutcomes() { + if r.flow == "deploy_status" && r.outcome == "leaked" { + deployLeaked = true + } + } + if !deployLeaked { + t.Error("deploy reap leak: want a deploy_status leaked outcome recorded") + } +} + +// TestFlowMoney_DeployReapHTTPError drives reapDeployment's transport-error +// branch: the deploy is accepted (201) but the deployment DELETE hijacks + +// closes the connection so httpCli.Do returns a transport error → leaked. +func TestFlowMoney_DeployReapHTTPError(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"commit_id":"abc1234"}`)) + }) + mux.HandleFunc("/auth/me", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"email":"synthetic+flowtest@instanode.dev"}`)) + }) + mux.HandleFunc("/db/new", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"11111111-1111-4111-8111-111111111111"}`)) + }) + mux.HandleFunc("/api/v1/resources/", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) // provision_reap reaps cleanly + }) + mux.HandleFunc("/claim/preview", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"invalid_token"}`)) + }) + mux.HandleFunc("/auth/email/start", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"ok":true}`)) + }) + mux.HandleFunc("/deploy/new", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"dep-33333333-3333-4333-8333-333333333333"}`)) + }) + mux.HandleFunc("/api/v1/billing/checkout", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"short_url":"https://rzp.example/x"}`)) + }) + // The deployment DELETE hijacks + closes → client sees a transport error. + mux.HandleFunc("/api/v1/deployments/", func(w http.ResponseWriter, _ *http.Request) { + hj, ok := w.(http.Hijacker) + if !ok { + w.WriteHeader(http.StatusInternalServerError) + return + } + conn, _, err := hj.Hijack() + if err == nil { + _ = conn.Close() + } + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + f := newFixtureCfg(t, srv, enabledConfig(srv)) + defer f.done() + expectSeed(f.mock) + expectReapAudit(f.mock) // provision_reap inline reap + expectForwarderClassification(f.mock, "success") // magic_link + expectReapAudit(f.mock) // deploy_status leaked ledger row (transport error) + expectOrphanSweepEmpty(f.mock) + + f.run(t) + var deployLeaked bool + for _, r := range f.fm.reapOutcomes() { + if r.flow == "deploy_status" && r.outcome == "leaked" { + deployLeaked = true + } + } + if !deployLeaked { + t.Error("deploy reap transport-error: want a deploy_status leaked outcome") + } +} + +// TestFlowMoney_Deploy4xxNon402 drives flowDeployStatus's "4xx other than 402" +// branch (e.g. a 400 missing-tarball on a headroom tier) → the gate/validation +// responded, non-5xx contract held → pass. +func TestFlowMoney_Deploy4xxNon402(t *testing.T) { + st := happyFlowState() + st.deployStatus = http.StatusBadRequest + st.deployBody = `{"error":"missing_tarball"}` + srv := newFlowAPIServer(st) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") + expectOrphanSweepEmpty(f.mock) + + f.run(t) + if got := f.fm.resultFor("deploy_status"); got != analyticsevent.ResultPass { + t.Errorf("deploy 4xx-non-402: want pass (non-5xx contract held), got %q", got) + } +} + +// TestFlowMoney_Deploy5xxFails drives flowDeployStatus's 5xx fail branch (the +// deploy endpoint crashed) → fail + audit row. +func TestFlowMoney_Deploy5xxFails(t *testing.T) { + st := happyFlowState() + st.deployStatus = http.StatusInternalServerError + st.deployBody = `{"error":"boom"}` + srv := newFlowAPIServer(st) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") + f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // deploy_status fail + expectOrphanSweepEmpty(f.mock) + + f.run(t) + if got := f.fm.resultFor("deploy_status"); got != analyticsevent.ResultFail { + t.Errorf("deploy 5xx: want fail, got %q", got) + } +} + +// TestFlowMoney_Checkout5xxFails drives flowCheckout's 5xx fail branch (the +// checkout handler crashed/regressed) → fail + audit row. +func TestFlowMoney_Checkout5xxFails(t *testing.T) { + st := happyFlowState() + st.checkoutStatus = http.StatusInternalServerError + st.checkoutBody = `{"error":"panic"}` + srv := newFlowAPIServer(st) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") + f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // checkout fail + expectOrphanSweepEmpty(f.mock) + + f.run(t) + if got := f.fm.resultFor("checkout"); got != analyticsevent.ResultFail { + t.Errorf("checkout 5xx: want fail, got %q", got) + } +} + +// TestFlowMoney_CheckoutBlocked4xx asserts a Razorpay-blocked 402/409 (non-5xx) +// from checkout passes (contract-only — alive-but-blocked is acceptable). +func TestFlowMoney_CheckoutBlocked4xx(t *testing.T) { + st := happyFlowState() + st.checkoutStatus = http.StatusBadGateway // 502 is a non-5xx? no — 502 IS 5xx + // Use a 402 to represent the operator-blocked-but-alive shape. + st.checkoutStatus = http.StatusPaymentRequired + st.checkoutBody = `{"error":"recurring_not_enabled"}` + srv := newFlowAPIServer(st) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") + expectOrphanSweepEmpty(f.mock) + + f.run(t) + if got := f.fm.resultFor("checkout"); got != analyticsevent.ResultPass { + t.Errorf("checkout blocked-4xx: want pass (alive-but-blocked), got %q", got) + } +} + +// TestFlowMoney_ClaimWrongError drives flowClaimPreview's "400 but unexpected +// error" branch: a 400 whose body is not invalid_token/missing_token is itself +// a contract regression → fail. +func TestFlowMoney_ClaimWrongError(t *testing.T) { + st := happyFlowState() + st.claimStatus = http.StatusBadRequest + st.claimBody = `{"error":"some_other_thing"}` + srv := newFlowAPIServer(st) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // claim fail + expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") + expectOrphanSweepEmpty(f.mock) + + f.run(t) + if got := f.fm.resultFor("claim"); got != analyticsevent.ResultFail { + t.Errorf("claim wrong-error: want fail, got %q", got) + } +} + +// TestFlowMoney_ClaimWrongStatus drives flowClaimPreview's non-400 branch (a +// 200 instead of the 400 contract means the claim path stopped validating). +func TestFlowMoney_ClaimWrongStatus(t *testing.T) { + st := happyFlowState() + st.claimStatus = http.StatusOK + st.claimBody = `{"token_valid":true}` + srv := newFlowAPIServer(st) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // claim fail + expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") + expectOrphanSweepEmpty(f.mock) + + f.run(t) + if got := f.fm.resultFor("claim"); got != analyticsevent.ResultFail { + t.Errorf("claim wrong-status: want fail, got %q", got) + } +} + +// TestFlowMoney_MagicLinkRejected drives flowMagicLink's failure-classification +// branch: the 202 succeeds but the forwarder_sent truth surface reports +// classification=rejected (the Brevo-unvalidated-sender reality) → fail. This is +// the rule-12 surface: a 202 is NOT a delivery, the ledger is. +func TestFlowMoney_MagicLinkRejected(t *testing.T) { + srv := newFlowAPIServer(happyFlowState()) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "rejected") // the truth surface + f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // magic_link fail + expectOrphanSweepEmpty(f.mock) + + f.run(t) + if got := f.fm.resultFor("magic_link"); got != analyticsevent.ResultFail { + t.Errorf("magic_link rejected: want fail (rule-12 truth surface), got %q", got) + } +} + +// TestFlowMoney_MagicLinkNoRow drives flowMagicLink's async-no-verdict branch: +// the 202 succeeds and there is no recent forwarder_sent row yet → the leg +// passes on the request-accepted state (the forwarder is async). +func TestFlowMoney_MagicLinkNoRow(t *testing.T) { + srv := newFlowAPIServer(happyFlowState()) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + expectReapAudit(f.mock) + expectForwarderNoRow(f.mock) // no recent send → no verdict + expectOrphanSweepEmpty(f.mock) + + f.run(t) + if got := f.fm.resultFor("magic_link"); got != analyticsevent.ResultPass { + t.Errorf("magic_link no-row: want pass (async, request accepted), got %q", got) + } +} + +// TestFlowMoney_MagicLinkNon202 drives flowMagicLink's non-202 fail branch (the +// send-request path itself is broken). No forwarder read happens. +func TestFlowMoney_MagicLinkNon202(t *testing.T) { + st := happyFlowState() + st.emailStartStat = http.StatusInternalServerError + st.emailStartBody = `{"error":"boom"}` + srv := newFlowAPIServer(st) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + expectSeed(f.mock) + expectReapAudit(f.mock) + f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // magic_link fail + expectOrphanSweepEmpty(f.mock) + + f.run(t) + if got := f.fm.resultFor("magic_link"); got != analyticsevent.ResultFail { + t.Errorf("magic_link non-202: want fail, got %q", got) + } +} + +// TestFlowMoney_MagicLinkNoRowDegraded drives the no-row + over-budget branch: +// 202 accepted, no forwarder row, but latency over the (0) budget → degraded. +func TestFlowMoney_MagicLinkNoRowDegraded(t *testing.T) { + srv := newFlowAPIServer(happyFlowState()) + defer srv.Close() + + f := newFixture(t, srv) + defer f.done() + f.w.SetBudgetOverrideForTest(map[string]time.Duration{"magic_link": 0}) + expectSeed(f.mock) + expectReapAudit(f.mock) + expectForwarderNoRow(f.mock) + expectOrphanSweepEmpty(f.mock) + + f.run(t) + if got := f.fm.resultFor("magic_link"); got != "degraded" { + t.Errorf("magic_link no-row degraded: want degraded, got %q", got) + } +} + +// TestFlowMoney_ClassifierSeam covers the exported classifier test seam + +// the healthy-vs-failure partition (case-insensitive + trimmed). +func TestFlowMoney_ClassifierSeam(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {"rejected", true}, + {" REJECTED ", true}, // trimmed + case-insensitive + {"bounced_hard", true}, + {"error", true}, + {"success", false}, + {"delivered", false}, + {"", false}, + } + for _, c := range cases { + if got := jobs.IsForwarderFailureClassificationForTest(c.in); got != c.want { + t.Errorf("IsForwarderFailureClassification(%q): want %v, got %v", c.in, c.want, got) + } + } +} diff --git a/internal/jobs/flow_synthetic_test.go b/internal/jobs/flow_synthetic_test.go index a129d5e..b6fb157 100644 --- a/internal/jobs/flow_synthetic_test.go +++ b/internal/jobs/flow_synthetic_test.go @@ -18,6 +18,7 @@ package jobs_test import ( "context" + "database/sql" "errors" "net/http" "net/http/httptest" @@ -140,6 +141,17 @@ type flowAPIState struct { dbNewStatus int dbNewBody string deleteStatus int + + // Money/value-journey legs (flow_synthetic_money.go). + claimStatus int // GET /claim/preview status (contract: 400) + claimBody string // GET /claim/preview body (contract: {"error":"invalid_token"}) + deployStatus int // POST /deploy/new status (free-tier contract: 402) + deployBody string // POST /deploy/new body + deployDelStat int // DELETE /api/v1/deployments/:id status (reap path) + checkoutStatus int // POST /api/v1/billing/checkout status (contract: non-5xx) + checkoutBody string // POST /api/v1/billing/checkout body + emailStartStat int // POST /auth/email/start status (contract: 202) + emailStartBody string // POST /auth/email/start body } func newFlowAPIServer(st *flowAPIState) *httptest.Server { @@ -160,6 +172,26 @@ func newFlowAPIServer(st *flowAPIState) *httptest.Server { mux.HandleFunc("/api/v1/resources/", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(st.deleteStatus) }) + // Money/value-journey routes. + mux.HandleFunc("/claim/preview", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(st.claimStatus) + _, _ = w.Write([]byte(st.claimBody)) + }) + mux.HandleFunc("/deploy/new", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(st.deployStatus) + _, _ = w.Write([]byte(st.deployBody)) + }) + mux.HandleFunc("/api/v1/deployments/", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(st.deployDelStat) + }) + mux.HandleFunc("/api/v1/billing/checkout", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(st.checkoutStatus) + _, _ = w.Write([]byte(st.checkoutBody)) + }) + mux.HandleFunc("/auth/email/start", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(st.emailStartStat) + _, _ = w.Write([]byte(st.emailStartBody)) + }) return httptest.NewServer(mux) } @@ -173,6 +205,17 @@ func happyFlowState() *flowAPIState { dbNewStatus: http.StatusCreated, dbNewBody: `{"ok":true,"id":"11111111-1111-4111-8111-111111111111","token":"22222222-2222-4222-8222-222222222222"}`, deleteStatus: http.StatusOK, + + // Money/value legs — happy contract defaults. + claimStatus: http.StatusBadRequest, // claim preview contract: bad token → 400 + claimBody: `{"error":"invalid_token"}`, + deployStatus: http.StatusPaymentRequired, // free-tier deploy gate contract: 402 + deployBody: `{"error":"over_limit"}`, + deployDelStat: http.StatusOK, + checkoutStatus: http.StatusOK, // checkout reachable, non-5xx + checkoutBody: `{"short_url":"https://rzp.example/x"}`, + emailStartStat: http.StatusAccepted, // magic-link send accepted: 202 + emailStartBody: `{"ok":true}`, } } @@ -209,6 +252,23 @@ func expectOrphanSweepEmpty(mock sqlmock.Sqlmock) { WillReturnRows(sqlmock.NewRows([]string{"id"})) } +// expectForwarderClassification sets the expectation for the magic_link leg's +// rule-12 truth-surface read (latestForwarderClassification). Pass the +// classification the synthetic recipient's latest send resolved to ("success", +// "rejected", …). Call once per seeded Work() (the magic_link leg runs only +// when the DB is present and the leg is not killed). +func expectForwarderClassification(mock sqlmock.Sqlmock, classification string) { + mock.ExpectQuery(`SELECT classification\s+FROM forwarder_sent`). + WillReturnRows(sqlmock.NewRows([]string{"classification"}).AddRow(classification)) +} + +// expectForwarderNoRow sets the expectation for the magic_link leg finding no +// recent forwarder_sent row (the async-no-verdict case → leg passes on the 202). +func expectForwarderNoRow(mock sqlmock.Sqlmock) { + mock.ExpectQuery(`SELECT classification\s+FROM forwarder_sent`). + WillReturnError(sql.ErrNoRows) +} + // flowFixture bundles a worker wired against an sqlmock DB + httptest server, // plus the metric + event capture fakes and the mock for setting expectations. type flowFixture struct { @@ -292,12 +352,13 @@ func TestFlowSynthetic_HappyPath_AllFlowsPass(t *testing.T) { f := newFixture(t, srv) defer f.done() expectSeed(f.mock) - expectReapAudit(f.mock) // the provision→reap leg's synthetic.reaped row + expectReapAudit(f.mock) // the provision→reap leg's synthetic.reaped row + expectForwarderClassification(f.mock, "success") // magic_link rule-12 truth surface expectOrphanSweepEmpty(f.mock) f.run(t) - for _, flow := range []string{"healthz", "auth_me", "provision_reap"} { + for _, flow := range []string{"healthz", "auth_me", "provision_reap", "claim", "magic_link", "deploy_status", "checkout"} { if got := f.fm.resultFor(flow); got != analyticsevent.ResultPass { t.Errorf("flow %s: want result=pass, got %q", flow, got) } @@ -316,9 +377,11 @@ func TestFlowSynthetic_HappyPath_AllFlowsPass(t *testing.T) { t.Errorf("want 1 reaped resource, got %d", reaped) } // One InstantFlowTest event per flow, all cohort=synthetic + commitId. + // 7 flows: 3 P0 (healthz/auth_me/provision_reap) + 4 money + // (claim/magic_link/deploy_status/checkout). evs := f.fe.flowEvents() - if len(evs) != 3 { - t.Fatalf("want 3 events, got %d", len(evs)) + if len(evs) != 7 { + t.Fatalf("want 7 events, got %d", len(evs)) } for _, e := range evs { if e.eventType != analyticsevent.EventFlowTest { @@ -348,7 +411,8 @@ func TestFlowSynthetic_HealthzDown_FailsAndAudits(t *testing.T) { expectSeed(f.mock) // healthz fail → audit row for flow_test_failed. f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) - expectReapAudit(f.mock) // provision→reap still reaps + expectReapAudit(f.mock) // provision→reap still reaps + expectForwarderClassification(f.mock, "success") // magic_link truth surface expectOrphanSweepEmpty(f.mock) f.run(t) @@ -372,6 +436,7 @@ func TestFlowSynthetic_ProvisionReapLeak_Fails(t *testing.T) { expectSeed(f.mock) expectReapAudit(f.mock) // the leaked reap still writes a ledger row f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // flow_test_failed for the leak + expectForwarderClassification(f.mock, "success") // magic_link truth surface expectOrphanSweepEmpty(f.mock) f.run(t) @@ -403,6 +468,7 @@ func TestFlowSynthetic_ProvisionFails_NoReap(t *testing.T) { defer f.done() expectSeed(f.mock) f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // provision fail audit + expectForwarderClassification(f.mock, "success") // magic_link truth surface expectOrphanSweepEmpty(f.mock) f.run(t) @@ -430,6 +496,7 @@ func TestFlowSynthetic_PerFlowKillSwitch_Degrades(t *testing.T) { defer f.done() expectSeed(f.mock) expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") // magic_link truth surface expectOrphanSweepEmpty(f.mock) f.run(t) @@ -453,7 +520,8 @@ func TestFlowSynthetic_NoJWTSecret_AuthedFlowsDegrade(t *testing.T) { cfg.JWTSecret = "" // the field under test f := newFixtureCfg(t, srv, cfg) defer f.done() - expectSeed(f.mock) // seed still runs (DB present); only the mint fails + expectSeed(f.mock) // seed still runs (DB present); only the mint fails + expectForwarderClassification(f.mock, "success") // claim+magic_link still run (no bearer needed); deploy/checkout degrade expectOrphanSweepEmpty(f.mock) f.run(t) @@ -461,11 +529,17 @@ func TestFlowSynthetic_NoJWTSecret_AuthedFlowsDegrade(t *testing.T) { if got := f.fm.resultFor("healthz"); got != analyticsevent.ResultPass { t.Errorf("anon healthz should pass: got %q", got) } - for _, flow := range []string{"auth_me", "provision_reap"} { + for _, flow := range []string{"auth_me", "provision_reap", "deploy_status", "checkout"} { if got := f.fm.resultFor(flow); got != "degraded" { t.Errorf("flow %s with no JWT: want degraded, got %q", flow, got) } } + // claim (anon) + magic_link (DB-only) still run despite the mint failure. + for _, flow := range []string{"claim", "magic_link"} { + if got := f.fm.resultFor(flow); got != analyticsevent.ResultPass { + t.Errorf("flow %s should still run without a JWT: want pass, got %q", flow, got) + } + } } // TestFlowSynthetic_OrphanSweep_ReapsBackstop asserts the reaper backstop sweeps @@ -477,7 +551,8 @@ func TestFlowSynthetic_OrphanSweep_ReapsBackstop(t *testing.T) { f := newFixture(t, srv) defer f.done() expectSeed(f.mock) - expectReapAudit(f.mock) // provision→reap inline + expectReapAudit(f.mock) // provision→reap inline + expectForwarderClassification(f.mock, "success") // magic_link truth surface (before the orphan sweep) // Orphan sweep finds one stale resource → UPDATE + ledger row. f.mock.ExpectQuery(`SELECT id::text\s+FROM resources`). WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow("99999999-9999-4999-8999-999999999999")) @@ -509,6 +584,7 @@ func TestFlowSynthetic_SeedFails_AuthedFlowsDegrade(t *testing.T) { f.mock.ExpectBegin() f.mock.ExpectExec(`INSERT INTO teams`).WillReturnError(errSeed) f.mock.ExpectRollback() + expectForwarderClassification(f.mock, "success") // claim+magic_link still run despite seed failure expectOrphanSweepEmpty(f.mock) f.run(t) @@ -516,7 +592,7 @@ func TestFlowSynthetic_SeedFails_AuthedFlowsDegrade(t *testing.T) { if got := f.fm.resultFor("healthz"); got != analyticsevent.ResultPass { t.Errorf("anon healthz should still pass on seed failure: got %q", got) } - for _, flow := range []string{"auth_me", "provision_reap"} { + for _, flow := range []string{"auth_me", "provision_reap", "deploy_status", "checkout"} { if got := f.fm.resultFor(flow); got != "degraded" { t.Errorf("flow %s on seed failure: want degraded, got %q", flow, got) } @@ -545,6 +621,7 @@ func TestFlowSynthetic_HealthzBadBody_Fails(t *testing.T) { expectSeed(f.mock) f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // healthz fail expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") expectOrphanSweepEmpty(f.mock) f.run(t) @@ -580,6 +657,7 @@ func TestFlowSynthetic_AuthMeBadBody_Fails(t *testing.T) { expectSeed(f.mock) f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // auth_me fail expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") expectOrphanSweepEmpty(f.mock) f.run(t) @@ -611,6 +689,7 @@ func TestFlowSynthetic_ProvisionBadBody_Fails(t *testing.T) { defer f.done() expectSeed(f.mock) f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // provision fail + expectForwarderClassification(f.mock, "success") // magic_link truth surface expectOrphanSweepEmpty(f.mock) f.run(t) @@ -628,7 +707,7 @@ func TestFlowSynthetic_AllFlowsKilled(t *testing.T) { defer srv.Close() cfg := enabledConfig(srv) - cfg.DisabledFlows = []string{"healthz", "auth_me", "provision_reap"} + cfg.DisabledFlows = []string{"healthz", "auth_me", "provision_reap", "claim", "magic_link", "deploy_status", "checkout"} f := newFixtureCfg(t, srv, cfg) defer f.done() expectSeed(f.mock) @@ -636,7 +715,7 @@ func TestFlowSynthetic_AllFlowsKilled(t *testing.T) { f.run(t) - for _, flow := range []string{"healthz", "auth_me", "provision_reap"} { + for _, flow := range []string{"healthz", "auth_me", "provision_reap", "claim", "magic_link", "deploy_status", "checkout"} { if got := f.fm.resultFor(flow); got != "degraded" { t.Errorf("killed flow %s: want degraded, got %q", flow, got) } @@ -708,6 +787,7 @@ func TestFlowSynthetic_AuditInsertError_NonFatal(t *testing.T) { expectSeed(f.mock) f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnError(errSeed) // audit write fails expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") // magic_link truth surface expectOrphanSweepEmpty(f.mock) f.run(t) // must not panic / error @@ -726,6 +806,7 @@ func TestFlowSynthetic_OrphanQueryError_NonFatal(t *testing.T) { defer f.done() expectSeed(f.mock) expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") // magic_link truth surface (before the orphan sweep) f.mock.ExpectQuery(`SELECT id::text\s+FROM resources`).WillReturnError(errSeed) f.run(t) // must not panic @@ -782,14 +863,17 @@ func TestFlowSynthetic_BadBaseURL_BuildRequestFails(t *testing.T) { f := newFixtureCfg(t, srv, cfg) defer f.done() expectSeed(f.mock) - // healthz + auth_me + provision all fail build_request → 3 audit rows. - f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) - f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) - f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) + // Every flow fails build_request (the bad URL never parses) → one audit row + // each. P0: healthz, auth_me, provision_reap. Money: claim, magic_link, + // deploy_status, checkout. The magic_link leg fails on build_request BEFORE + // the forwarder read, so there is no forwarder query in this case. 7 rows. + for i := 0; i < 7; i++ { + f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) + } expectOrphanSweepEmpty(f.mock) f.run(t) - for _, flow := range []string{"healthz", "auth_me", "provision_reap"} { + for _, flow := range []string{"healthz", "auth_me", "provision_reap", "claim", "magic_link", "deploy_status", "checkout"} { if got := f.fm.resultFor(flow); got != analyticsevent.ResultFail { t.Errorf("bad URL: flow %s want fail, got %q", flow, got) } @@ -811,13 +895,16 @@ func TestFlowSynthetic_HTTPError_AllFlowsFail(t *testing.T) { f := newFixtureCfg(t, srv, cfg) defer f.done() expectSeed(f.mock) - f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) - f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) - f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) + // All 7 flows hit a dead address → http_error fail → one audit row each. + // magic_link fails on the http_error before the forwarder read, so no + // forwarder query here. + for i := 0; i < 7; i++ { + f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) + } expectOrphanSweepEmpty(f.mock) f.run(t) - for _, flow := range []string{"healthz", "auth_me", "provision_reap"} { + for _, flow := range []string{"healthz", "auth_me", "provision_reap", "claim", "magic_link", "deploy_status", "checkout"} { if got := f.fm.resultFor(flow); got != analyticsevent.ResultFail { t.Errorf("http_error: flow %s want fail, got %q", flow, got) } @@ -835,13 +922,15 @@ func TestFlowSynthetic_DegradedLatency(t *testing.T) { defer f.done() f.w.SetBudgetOverrideForTest(map[string]time.Duration{ "healthz": 0, "auth_me": 0, "provision_reap": 0, + "claim": 0, "magic_link": 0, "deploy_status": 0, "checkout": 0, }) expectSeed(f.mock) - expectReapAudit(f.mock) // provision_reap still reaps before the degraded check + expectReapAudit(f.mock) // provision_reap still reaps before the degraded check + expectForwarderClassification(f.mock, "success") // magic_link reads classification, then degrades on budget expectOrphanSweepEmpty(f.mock) f.run(t) - for _, flow := range []string{"healthz", "auth_me", "provision_reap"} { + for _, flow := range []string{"healthz", "auth_me", "provision_reap", "claim", "magic_link", "deploy_status", "checkout"} { if got := f.fm.resultFor(flow); got != "degraded" { t.Errorf("0-budget: flow %s want degraded, got %q", flow, got) } @@ -883,6 +972,13 @@ func TestFlowSynthetic_ReapHTTPError(t *testing.T) { expectSeed(f.mock) expectReapAudit(f.mock) // leaked ledger row f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // flow_test_failed (leak) + // This custom mux does NOT serve the money routes, so claim (404≠400) and + // magic_link (404≠202) fail → one flow_test_failed audit row each. deploy + // (404 is 4xx → non-5xx contract held → pass) and checkout (404<500 → pass) + // do not audit. magic_link fails on the 404 before the forwarder read, so no + // forwarder query. + f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // claim fail + f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // magic_link fail expectOrphanSweepEmpty(f.mock) f.run(t) @@ -936,6 +1032,7 @@ func TestFlowSynthetic_SeedSubErrors(t *testing.T) { f := newFixture(t, srv) defer f.done() s.setup(f.mock) + expectForwarderClassification(f.mock, "success") // claim+magic_link still run despite seed sub-failure expectOrphanSweepEmpty(f.mock) f.run(t) @@ -953,6 +1050,7 @@ func TestFlowSynthetic_SeedBeginError(t *testing.T) { f := newFixture(t, srv) defer f.done() f.mock.ExpectBegin().WillReturnError(errSeed) + expectForwarderClassification(f.mock, "success") // claim+magic_link still run despite seed-begin failure expectOrphanSweepEmpty(f.mock) f.run(t) @@ -970,6 +1068,7 @@ func TestFlowSynthetic_OrphanUpdateError(t *testing.T) { defer f.done() expectSeed(f.mock) expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") // magic_link truth surface (before the orphan sweep) f.mock.ExpectQuery(`SELECT id::text\s+FROM resources`). WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow("88888888-8888-4888-8888-888888888888")) f.mock.ExpectExec(`UPDATE resources SET status = 'deleted'`).WillReturnError(errSeed) @@ -998,6 +1097,7 @@ func TestFlowSynthetic_OrphanScanError(t *testing.T) { defer f.done() expectSeed(f.mock) expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") // magic_link truth surface (before the orphan sweep) // Two columns vs the single Scan(&id) destination → Scan error per row. rows := sqlmock.NewRows([]string{"id", "extra"}).AddRow("77777777-7777-4777-8777-777777777777", "x") f.mock.ExpectQuery(`SELECT id::text\s+FROM resources`).WillReturnRows(rows) @@ -1014,6 +1114,7 @@ func TestFlowSynthetic_OrphanRowsError(t *testing.T) { defer f.done() expectSeed(f.mock) expectReapAudit(f.mock) + expectForwarderClassification(f.mock, "success") // magic_link truth surface (before the orphan sweep) rows := sqlmock.NewRows([]string{"id"}).AddRow("66666666-6666-4666-8666-666666666666").RowError(0, errSeed) f.mock.ExpectQuery(`SELECT id::text\s+FROM resources`).WillReturnRows(rows) @@ -1063,6 +1164,7 @@ func TestFlowSynthetic_ReapAuditInsertError(t *testing.T) { expectSeed(f.mock) // provision_reap inline reap → ledger INSERT fails (non-fatal). f.mock.ExpectExec(`INSERT INTO audit_log`).WillReturnError(errSeed) + expectForwarderClassification(f.mock, "success") // magic_link truth surface expectOrphanSweepEmpty(f.mock) f.run(t) // must not panic; provision_reap still passes (reap succeeded HTTP-side)