From f1820f3ab58a75d59fa6581007a82f9f7bae12e0 Mon Sep 17 00:00:00 2001 From: os-zhuang Date: Thu, 18 Jun 2026 13:29:09 +0800 Subject: [PATCH] feat(dogfood): add in-process runtime regression gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Static gates (build, unit tests, spec-liveness, CodeQL) verify each layer in isolation, usually against mocks, and cannot catch a break that only appears when the real engine + strategies + services + HTTP context run together. The #2018 tz-bucketing regression is the canonical case: green on every static gate (900+ unit tests included) because each of its three broken seams was individually correct and individually mocked. This adds `@objectstack/dogfood` (private), a harness that boots real example apps in-process against in-memory SQLite — wired with the same service plugins `objectstack dev` loads — and exercises them through the real HTTP surface via Hono request-injection (no ports, no sockets, CI-stable). Tests act as a browser client: sign in, hit /api/v1/..., assert on real responses. - src/harness.ts — bootDogfoodStack(config) → { kernel, api, raw, signIn, apiAs, stop } - test/analytics-timezone.dogfood.test.ts — golden regression for #1982/#2018: org timezone (set via the real settings route) must shift a boundary-crossing instant's date bucket (UTC 2024-03-01 → America/Los_Angeles 2024-02-29) with the correct count. Verified to FAIL when any #2018 seam is reverted. - CI: new "Dogfood Regression Gate" job (also runs under turbo run test). Boot-to-assert ~2s. Private package — no changeset. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 55 +++++++ packages/dogfood/README.md | 39 +++++ packages/dogfood/package.json | 30 ++++ packages/dogfood/src/harness.ts | 149 ++++++++++++++++++ .../test/analytics-timezone.dogfood.test.ts | 103 ++++++++++++ packages/dogfood/tsconfig.json | 15 ++ pnpm-lock.yaml | 49 ++++++ 7 files changed, 440 insertions(+) create mode 100644 packages/dogfood/README.md create mode 100644 packages/dogfood/package.json create mode 100644 packages/dogfood/src/harness.ts create mode 100644 packages/dogfood/test/analytics-timezone.dogfood.test.ts create mode 100644 packages/dogfood/tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9230270f92..08defeebce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,61 @@ jobs: path: packages/spec/coverage/ retention-days: 30 + dogfood: + name: Dogfood Regression Gate + needs: filter + if: needs.filter.outputs.core == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '20' + + - name: Enable Corepack + run: corepack enable + + - name: Verify pnpm version + run: pnpm --version + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Setup pnpm cache + uses: actions/cache@v5 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store-v3- + + - name: Setup Turbo cache + uses: actions/cache@v5 + with: + path: .turbo/cache + key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}- + ${{ runner.os }}-turbo-${{ github.job }}- + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # Boots real example apps in-process (in-memory SQLite) and exercises them + # through the real HTTP + service stack — catches runtime regressions that + # build / unit tests / spec-liveness pass over (e.g. the #2018 tz-bucketing + # break, which was green on every static gate). + - name: Boot example apps and exercise real user flows + run: pnpm turbo run test --filter=@objectstack/dogfood + build-core: name: Build Core needs: filter diff --git a/packages/dogfood/README.md b/packages/dogfood/README.md new file mode 100644 index 0000000000..4febd3ea6c --- /dev/null +++ b/packages/dogfood/README.md @@ -0,0 +1,39 @@ +# @objectstack/dogfood + +Runtime regression gate. Private (never published). + +## Why + +Static gates — `build`, unit tests, spec-liveness, CodeQL — verify each layer in +isolation, usually against mocks. They cannot catch a break that only appears +when the **real engine + strategies + services + HTTP context run together**. + +The canonical example is [#2018](https://github.com/objectstack-ai/framework/pull/2018): +"organization timezone drives analytics date bucketing" was broken across three +seams (analytics strategy routing, in-memory count, REST execution-context). +Every static gate was green — 900+ unit tests included — because each layer was +individually correct and individually mocked. The bug was only visible by +booting the app and comparing a date bucket under UTC vs a non-UTC org timezone. + +This package boots **real example apps in-process** (in-memory SQLite), wired +with the same service plugins `objectstack dev` loads, and exercises them +through the **real HTTP surface** (Hono request-injection — no ports, no +sockets, CI-stable). Tests act as a browser client would: sign in, hit +`/api/v1/...`, assert on real responses. + +## Layout + +- `src/harness.ts` — `bootDogfoodStack(config)` → `{ kernel, api, raw, signIn, apiAs, stop }`. +- `test/*.dogfood.test.ts` — golden flows. Each should assert on **observable + output** (a number, a bucket label, a row count), not just "no error". + +## Adding a golden test + +1. Pick a real user flow that a static test can't cover (it spans engine + + service + HTTP, or depends on seeded/written data). +2. `bootDogfoodStack()`, `signIn()`, drive it via `api()/apiAs()`. +3. Assert on the concrete result. +4. **Prove it catches the bug**: temporarily revert the relevant fix and confirm + the test goes red. A green-on-the-bug test is not a gate. + +Runs in CI as the `Dogfood Regression Gate` job (and under `turbo run test`). diff --git a/packages/dogfood/package.json b/packages/dogfood/package.json new file mode 100644 index 0000000000..5390f5d0f9 --- /dev/null +++ b/packages/dogfood/package.json @@ -0,0 +1,30 @@ +{ + "name": "@objectstack/dogfood", + "version": "0.0.0", + "private": true, + "license": "Apache-2.0", + "description": "Dogfood regression gate — boots real example apps in-process and exercises them through the real HTTP/service stack, catching runtime regressions that static checks (build, unit tests, spec liveness) miss.", + "type": "module", + "scripts": { + "test": "vitest run" + }, + "dependencies": { + "@objectstack/core": "workspace:*", + "@objectstack/runtime": "workspace:*", + "@objectstack/objectql": "workspace:*", + "@objectstack/spec": "workspace:*", + "@objectstack/driver-sqlite-wasm": "workspace:*", + "@objectstack/plugin-hono-server": "workspace:*", + "@objectstack/rest": "workspace:*", + "@objectstack/plugin-auth": "workspace:*", + "@objectstack/plugin-security": "workspace:*", + "@objectstack/service-settings": "workspace:*", + "@objectstack/service-analytics": "workspace:*", + "@objectstack/example-crm": "workspace:*" + }, + "devDependencies": { + "@types/node": "^25.9.3", + "typescript": "^6.0.3", + "vitest": "^4.1.9" + } +} diff --git a/packages/dogfood/src/harness.ts b/packages/dogfood/src/harness.ts new file mode 100644 index 0000000000..cf0858fa41 --- /dev/null +++ b/packages/dogfood/src/harness.ts @@ -0,0 +1,149 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// Dogfood boot harness. +// +// Boots a real ObjectStack app **in-process** against an in-memory SQLite +// database, wired with the same service plugins `objectstack dev` loads, and +// exposes the live HTTP surface via Hono's request-injection (no port, no +// sockets — CI-stable). Tests then exercise the app exactly as a browser +// client would: sign in, hit `/api/v1/...`, assert on real responses. +// +// Why this exists: the bucketing regression fixed in #2018 passed every static +// gate (build, 900+ unit tests, spec-liveness, CodeQL) because each layer was +// individually correct and individually mocked — the break only appeared when +// the real engine + strategy + settings + REST context ran together. Unit +// tests that mock the protocol/server (e.g. rest.test.ts) cannot catch that. +// This harness runs the integrated stack so they can. + +import { ObjectKernel, AppPlugin, DriverPlugin, createDispatcherPlugin } from '@objectstack/runtime'; +import { ObjectQLPlugin } from '@objectstack/objectql'; +import { SqliteWasmDriver } from '@objectstack/driver-sqlite-wasm'; +import { HonoServerPlugin } from '@objectstack/plugin-hono-server'; +import { createRestApiPlugin } from '@objectstack/rest'; +import { AuthPlugin } from '@objectstack/plugin-auth'; +import { SecurityPlugin } from '@objectstack/plugin-security'; +import { SettingsServicePlugin } from '@objectstack/service-settings'; +import { AnalyticsServicePlugin } from '@objectstack/service-analytics'; + +/** A Hono app exposes `.request(path, init)` returning a standard `Response`. */ +interface InjectableApp { + request(input: string, init?: RequestInit): Promise; +} + +const API_PREFIX = '/api/v1'; +const DEFAULT_ADMIN_EMAIL = 'admin@objectos.ai'; +const DEFAULT_ADMIN_PASSWORD = 'admin123'; + +export interface DogfoodStack { + /** The booted kernel — for direct service calls when bypassing HTTP is intentional. */ + kernel: ObjectKernel; + /** Inject an HTTP request through the real Hono app (no socket). Path is relative to `/api/v1`. */ + api(path: string, init?: RequestInit): Promise; + /** Inject a request at an absolute path (e.g. `/api/settings/...`). */ + raw(path: string, init?: RequestInit): Promise; + /** Sign in through the real auth route; returns a bearer token. Defaults to the dev admin. */ + signIn(email?: string, password?: string): Promise; + /** Convenience: an authed JSON request relative to `/api/v1`. */ + apiAs(token: string, method: string, path: string, body?: unknown): Promise; + /** Tear down the kernel (close DB / HTTP handles). */ + stop(): Promise; +} + +export interface BootOptions { + /** Override the dev admin credentials the harness signs in with. */ + admin?: { email: string; password: string }; +} + +/** + * Boot an app config in-process and return a live dogfood stack. + * + * `NODE_ENV` is forced to `development` so the auth plugin's dev-admin + * bootstrap provisions a known, loginable admin (mirrors `objectstack dev`). + */ +export async function bootDogfoodStack( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + config: any, + opts: BootOptions = {}, +): Promise { + process.env.NODE_ENV = 'development'; + + const kernel = new ObjectKernel(); + + // Data engine + in-memory SQLite (pure-JS WASM driver — no native build, CI-safe). + await kernel.use(new ObjectQLPlugin()); + await kernel.use(new DriverPlugin(new SqliteWasmDriver({ filename: ':memory:' }))); + + // HTTP server (registers the `http-server` IHttpServer service the REST + + // dispatcher plugins mount their routes onto). Port 0 = ephemeral; we never + // hit the socket — requests are injected through the Hono app directly. + await kernel.use(new HonoServerPlugin({ port: 0 })); + + // The app under test (objects, datasets, cubes, flows, seed data). + await kernel.use(new AppPlugin(config)); + + // Service plugins `objectstack dev` auto-loads for an app of this shape. + await kernel.use(new SettingsServicePlugin()); + await kernel.use(new AnalyticsServicePlugin()); + await kernel.use(new AuthPlugin({ secret: 'dogfood-regression-secret' })); + await kernel.use(new SecurityPlugin()); + + // REST + dispatcher route surfaces (mount onto the http-server service). + await kernel.use(createRestApiPlugin({ api: { api: { requireAuth: true } } as never })); + await kernel.use(createDispatcherPlugin({})); + + // Fire the ready lifecycle: seed data, dev-admin bootstrap, route registration. + await kernel.bootstrap(); + + const httpServer = await kernel.getServiceAsync<{ getRawApp(): InjectableApp; close?(): Promise }>( + 'http-server', + ); + const app = httpServer.getRawApp(); + + const raw = (path: string, init?: RequestInit) => app.request(path, init); + const api = (path: string, init?: RequestInit) => raw(`${API_PREFIX}${path}`, init); + + const admin = opts.admin ?? { email: DEFAULT_ADMIN_EMAIL, password: DEFAULT_ADMIN_PASSWORD }; + + const signIn = async ( + email: string = admin.email, + password: string = admin.password, + ): Promise => { + const res = await api('/auth/sign-in/email', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email, password }), + }); + if (!res.ok) { + throw new Error(`dogfood signIn failed: ${res.status} ${await res.text()}`); + } + const data = (await res.json()) as { token?: string }; + if (!data.token) throw new Error('dogfood signIn: no token in response'); + return data.token; + }; + + const apiAs = (token: string, method: string, path: string, body?: unknown) => + api(path, { + method, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}`, + }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + + const stop = async () => { + try { + await httpServer.close?.(); + } catch { + /* best-effort */ + } + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await (kernel as any).shutdown?.(); + } catch { + /* best-effort */ + } + }; + + return { kernel, api, raw, signIn, apiAs, stop }; +} diff --git a/packages/dogfood/test/analytics-timezone.dogfood.test.ts b/packages/dogfood/test/analytics-timezone.dogfood.test.ts new file mode 100644 index 0000000000..f453eb057f --- /dev/null +++ b/packages/dogfood/test/analytics-timezone.dogfood.test.ts @@ -0,0 +1,103 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// GOLDEN REGRESSION for #1982 / #2018 — "organization timezone drives analytics +// date bucketing", exercised end-to-end through the real HTTP + service stack. +// +// This is the test that would have caught #2018 before merge. That bug passed +// every static gate: it lived in the *integration* of NativeSQLStrategy routing +// + in-memory count + REST execution-context resolution. Only booting the app +// and comparing UTC vs non-UTC buckets surfaces it. + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import crmStack from '@objectstack/example-crm'; +import { bootDogfoodStack, type DogfoodStack } from '../src/harness.js'; + +// 03:00 UTC on 2024-03-01 is still 2024-02-29 (19:00) in America/Los_Angeles +// (PST = UTC-8, before DST). So a *day* bucket labels this instant 2024-03-01 +// under UTC but 2024-02-29 under LA — the exact boundary behavior #2018 fixed. +const BOUNDARY = '2024-03-01T03:00:00.000Z'; +const N_LEADS = 3; + +const leadByDay = { + name: 'lead_by_day', + label: 'Leads by day', + object: 'crm_lead', + dimensions: [ + { name: 'created', label: 'Created', field: 'created_at', type: 'date', dateGranularity: 'day' }, + ], + measures: [{ name: 'cnt', label: 'Count', aggregate: 'count' }], +}; + +describe('dogfood: org timezone drives analytics date bucketing (#1982/#2018)', () => { + let stack: DogfoodStack; + let token: string; + + beforeAll(async () => { + stack = await bootDogfoodStack(crmStack); + + // Deterministic fixture: N leads pinned to the tz-boundary instant, inserted + // as system so the write path's defaults/validation don't fight the setup. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const ql = await stack.kernel.getServiceAsync('objectql'); + for (let i = 0; i < N_LEADS; i++) { + await ql.insert( + 'crm_lead', + { name: `tz-lead-${i}`, status: 'new', created_at: BOUNDARY }, + { context: { isSystem: true } }, + ); + } + // Sanity: confirm created_at actually persisted as the boundary instant + // (if a stamp hook overrode it the whole premise is void — fail loudly). + const mine = (await ql.find('crm_lead', { + where: { name: { $in: ['tz-lead-0', 'tz-lead-1', 'tz-lead-2'] } }, + context: { isSystem: true }, + })) as Array>; + expect(mine).toHaveLength(N_LEADS); + for (const r of mine) { + expect(new Date(r.created_at as string).toISOString()).toBe(BOUNDARY); + } + + token = await stack.signIn(); + }, 60_000); + + afterAll(async () => { + await stack?.stop(); + }); + + // Set the org-wide timezone via the real settings route (NO explicit per-query + // tz), then bucket — exercising settings → REST exec-context → analytics. + async function bucketsForOrgTz(tz: string): Promise> { + const put = await stack.raw('/api/settings/localization', { + method: 'PUT', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` }, + body: JSON.stringify({ timezone: tz }), + }); + expect(put.status).toBe(200); + + const res = await stack.apiAs(token, 'POST', '/analytics/dataset/query', { + dataset: leadByDay, + selection: { dimensions: ['created'], measures: ['cnt'] }, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { rows?: Array<{ created: string; cnt: number }> }; + const out: Record = {}; + for (const row of body.rows ?? []) out[row.created] = row.cnt; + return out; + } + + it('buckets the boundary instant on the UTC calendar day with the correct count', async () => { + const utc = await bucketsForOrgTz('UTC'); + // The 3 boundary leads land on 2024-03-01; cnt must be 3 (not 0 — the + // count-all `*` in-memory bug), and not split across raw timestamps. + expect(utc['2024-03-01']).toBe(N_LEADS); + expect(utc['2024-02-29']).toBeUndefined(); + }); + + it('shifts the bucket to the previous day under America/Los_Angeles', async () => { + const la = await bucketsForOrgTz('America/Los_Angeles'); + // The regression: before #2018 this stayed on 2024-03-01 (tz dropped) or + // returned cnt 0. The org timezone must move it to 2024-02-29. + expect(la['2024-02-29']).toBe(N_LEADS); + expect(la['2024-03-01']).toBeUndefined(); + }); +}); diff --git a/packages/dogfood/tsconfig.json b/packages/dogfood/tsconfig.json new file mode 100644 index 0000000000..15de196d35 --- /dev/null +++ b/packages/dogfood/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true, + "types": ["node"], + "rootDir": "." + }, + "include": ["src/**/*", "test/**/*"], + "exclude": ["node_modules"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dafa4d8b66..bb28941280 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -812,6 +812,55 @@ importers: specifier: ^6.0.3 version: 6.0.3 + packages/dogfood: + dependencies: + '@objectstack/core': + specifier: workspace:* + version: link:../core + '@objectstack/driver-sqlite-wasm': + specifier: workspace:* + version: link:../plugins/driver-sqlite-wasm + '@objectstack/example-crm': + specifier: workspace:* + version: link:../../examples/app-crm + '@objectstack/objectql': + specifier: workspace:* + version: link:../objectql + '@objectstack/plugin-auth': + specifier: workspace:* + version: link:../plugins/plugin-auth + '@objectstack/plugin-hono-server': + specifier: workspace:* + version: link:../plugins/plugin-hono-server + '@objectstack/plugin-security': + specifier: workspace:* + version: link:../plugins/plugin-security + '@objectstack/rest': + specifier: workspace:* + version: link:../rest + '@objectstack/runtime': + specifier: workspace:* + version: link:../runtime + '@objectstack/service-analytics': + specifier: workspace:* + version: link:../services/service-analytics + '@objectstack/service-settings': + specifier: workspace:* + version: link:../services/service-settings + '@objectstack/spec': + specifier: workspace:* + version: link:../spec + devDependencies: + '@types/node': + specifier: ^25.9.3 + version: 25.9.3 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.9 + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.2)(msw@2.14.6(@types/node@25.9.3)(typescript@6.0.3))(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) + packages/formula: dependencies: '@marcbachmann/cel-js':