refactor: remove JSON database backend and update db dump workflow (#560)

* refactor: remove JSON database backend [skip ci]

* fix: keep feedback data out of public dumps

* docs: sanitize backup restore placeholder

* fix: restore xz database dump compression

* docs: improve database dump instructions

Author: adibarra

.env.example

@@ -1,24 +1,19 @@
 # ╔══════════════════════════════════════════════════════════════════════════╗
 # ║  Local development                                                       ║
 # ║                                                                          ║
-# ║  Choose ONE option: JSON dump, TCP database, or HTTP database.           ║
+# ║  Choose ONE option: standard PostgreSQL or Neon.                          ║
 # ╚══════════════════════════════════════════════════════════════════════════╝
 
 # LAN / remote dev: set this machine's IP so the dev server accepts cross-origin requests
 # NEXT_DEV_ALLOWED_ORIGINS=10.112.9.49
 
-# Option A: JSON dump (no database needed)
-# Download a DB dump release, unzip it, and point this to the directory.
-# See README for setup instructions.
-# DUMP_DIR=./inferencex-dump/inferencex-dump-2026-03-30
-
-# Option B: TCP database (postgres.js — works with any Postgres instance)
+# Option A: Standard PostgreSQL (postgres.js TCP driver)
 # DATABASE_READONLY_URL=postgresql://postgres:postgres@localhost:5432/postgres
 # DATABASE_DRIVER=postgres
 # DATABASE_SSL=false
 
-# Option C: HTTP database (Neon serverless driver — requires a Neon-hosted instance)
-# DATABASE_READONLY_URL=
+# Option B: Neon (serverless HTTP driver)
+# DATABASE_READONLY_URL=postgresql://user:password@ep-example.us-east-1.aws.neon.tech/database
 # DATABASE_DRIVER=neon
 # DATABASE_SSL=true
 

.github/workflows/db-backup.yml

@@ -19,44 +19,47 @@ jobs:
     permissions:
       contents: write # Create the database dump release
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
-        with:
-          persist-credentials: false
-      - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
-      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
-        with:
-          node-version: '24'
-          cache: pnpm
-      - name: Install dependencies
-        run: pnpm install --filter @semianalysisai/inferencex-db...
+      - name: Dump and split database
         env:
-          CYPRESS_INSTALL_BINARY: '0'
-
-      - name: Dump database
-        env:
-          DATABASE_READONLY_URL: ${{ secrets.DATABASE_READONLY_URL }}
+          # Use primary; replica recovery can cancel long pg_dump snapshots.
+          DATABASE_URL: ${{ secrets.DATABASE_WRITE_URL }}
         run: |
           set -euo pipefail
-          DUMP_DIR="inferencex-dump-$(date -u +%Y-%m-%d)"
-          pnpm admin:db:dump "$DUMP_DIR"
-          echo "DUMP_DIR=${DUMP_DIR}" >> "$GITHUB_ENV"
-          echo "TAG=db-dump/$(date -u +%Y-%m-%d)" >> "$GITHUB_ENV"
+          DUMP_FILE="inferencex-$(date -u +%Y-%m-%d).dump.xz"
 
-      - name: Compress and split dump
-        run: |
-          set -euo pipefail
-          ( cd packages/db
-            tar -cf - "$DUMP_DIR" \
-              | xz -v -T0 --memlimit-compress=99% --lzma2=preset=9e,dict=192MiB \
-              | split -b 1900m -d -a 2 - "${GITHUB_WORKSPACE}/${DUMP_DIR}.tar.xz.part" )
-          echo "DUMP_GLOB=${DUMP_DIR}.tar.xz.part*" >> "$GITHUB_ENV"
+          docker run --rm --env DATABASE_URL \
+            postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 \
+            sh -ceu 'exec pg_dump \
+              --dbname="$DATABASE_URL" \
+              --format=custom \
+              --compress=none \
+              --no-owner \
+              --no-privileges \
+              --exclude-table-data=public.user_feedback \
+              --verbose' \
+            | xz -v -T0 --memlimit-compress=99% --lzma2=preset=9e,dict=192MiB \
+            | split -b 1900m -d -a 2 - "${DUMP_FILE}.part"
+
+          echo "DUMP_FILE=${DUMP_FILE}" >> "$GITHUB_ENV"
+          echo "DUMP_GLOB=${DUMP_FILE}.part*" >> "$GITHUB_ENV"
+          echo "TAG=db-dump/$(date -u +%Y-%m-%d)" >> "$GITHUB_ENV"
 
       - name: Create release
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
+          cat > release-notes.md <<'EOF'
+          Weekly InferenceX database dump.
+
+          Reassemble and decompress:
+          `cat *.dump.xz.part* | xz -d > inferencex.dump`
+
+          Restore to PostgreSQL or Neon:
+          `pg_restore --exit-on-error --clean --if-exists --no-owner --no-privileges --dbname="YOUR_TARGET_DATABASE_URL" inferencex.dump`
+          EOF
+
           # $DUMP_GLOB is intentionally unquoted so the shell expands it to every part file.
           gh release create "$TAG" $DUMP_GLOB \
             --title "DB Dump $(date -u +%Y-%m-%d)" \
-            --notes 'Weekly full database dump (xz-compressed, split into parts under 2 GiB). Reassemble: `cat *.tar.xz.part* | xz -d | tar -x`' \
+            --notes-file release-notes.md \
             --latest=false

README.md

@@ -55,30 +55,37 @@ cd InferenceX-app
 pnpm install
 ```
 
-### 2. Set Up Data Source
+### 2. Set Up a Database
 
-You can run the dashboard against either a live database or a static JSON dump. The JSON dump approach requires no database setup and is the easiest way to get started.
+The dashboard requires a live PostgreSQL database. Set `DATABASE_READONLY_URL` in `.env`; the application supports both standard PostgreSQL servers and Neon.
 
-#### Option A: JSON Dump (no database required, local dev only)
+#### Option A: Standard PostgreSQL
 
-Download the latest DB dump from [GitHub Releases](https://github.com/SemiAnalysisAI/InferenceX-app/releases), unpack it, and point `DUMP_DIR` at the directory. The dump is xz-compressed and split into one or more `.tar.xz.part*` files; reassemble them by piping `cat` through `xz`. This only works with `pnpm dev`; production builds require a live database.
+Use the `postgres` driver for a local or remotely hosted PostgreSQL server:
 
 ```bash
 cp .env.example .env
-
-# Download and unpack the latest dump (requires xz; `brew install xz` on macOS)
-gh release download db-dump/2026-03-30 -p 'inferencex-dump-*.tar.xz.part*'
-cat inferencex-dump-2026-03-30.tar.xz.part* | xz -d -T0 | tar -x
-
-# Add to .env
-echo 'DUMP_DIR=./inferencex-dump-2026-03-30' >> .env
+cat >> .env <<'EOF'
+DATABASE_READONLY_URL=postgresql://postgres:postgres@localhost:5432/postgres
+DATABASE_DRIVER=postgres
+DATABASE_SSL=false
+EOF
 ```
 
-Make sure `DATABASE_READONLY_URL` is not set (or is commented out) in your `.env`.
+Remote PostgreSQL servers use TLS by default. Omit `DATABASE_SSL` unless the server explicitly requires it to be disabled.
+
+#### Option B: Neon
 
-#### Option B: Live Database
+Set `DATABASE_READONLY_URL` to a Neon PostgreSQL connection string. Neon hosts use the serverless HTTP driver automatically; `DATABASE_DRIVER=neon` can be set explicitly.
 
-Set `DATABASE_READONLY_URL` in your `.env` to a Neon PostgreSQL connection string. See [`.env.example`](.env.example) for details.
+```bash
+cp .env.example .env
+cat >> .env <<'EOF'
+DATABASE_READONLY_URL=postgresql://user:password@ep-example.us-east-1.aws.neon.tech/database
+DATABASE_DRIVER=neon
+DATABASE_SSL=true
+EOF
+```
 
 ### 3. Run the Development Server
 

README_zh.md

@@ -55,30 +55,37 @@ cd InferenceX-app
 pnpm install
 ```
 
-### 2. 配置数据源
+### 2. 配置数据库
 
-仪表板可以对接实时数据库，也可以使用静态 JSON dump。JSON dump 方式无需配置数据库，是最简单的上手方式。
+仪表板需要实时 PostgreSQL 数据库。在 `.env` 中设置 `DATABASE_READONLY_URL`；应用同时支持标准 PostgreSQL 服务器和 Neon。
 
-#### 方式 A：JSON Dump（无需数据库，仅限本地开发）
+#### 方式 A：标准 PostgreSQL
 
-从 [GitHub Releases](https://github.com/SemiAnalysisAI/InferenceX-app/releases) 下载最新的数据库 dump，解包后将 `DUMP_DIR` 指向该目录。dump 以 xz 压缩并拆分为一个或多个 `.tar.xz.part*` 文件；用 `cat` 管道接 `xz` 重新组装。此方式仅适用于 `pnpm dev`；生产构建需要实时数据库。
+本地或远程托管的 PostgreSQL 服务器使用 `postgres` 驱动：
 
 ```bash
 cp .env.example .env
-
-# 下载并解包最新 dump（需要 xz；macOS 上执行 `brew install xz`）
-gh release download db-dump/2026-03-30 -p 'inferencex-dump-*.tar.xz.part*'
-cat inferencex-dump-2026-03-30.tar.xz.part* | xz -d -T0 | tar -x
-
-# 写入 .env
-echo 'DUMP_DIR=./inferencex-dump-2026-03-30' >> .env
+cat >> .env <<'EOF'
+DATABASE_READONLY_URL=postgresql://postgres:postgres@localhost:5432/postgres
+DATABASE_DRIVER=postgres
+DATABASE_SSL=false
+EOF
 ```
 
-确保 `.env` 中没有设置（或已注释掉）`DATABASE_READONLY_URL`。
+远程 PostgreSQL 服务器默认使用 TLS。除非服务器明确要求禁用 TLS，否则请省略 `DATABASE_SSL`。
+
+#### 方式 B：Neon
 
-#### 方式 B：实时数据库
+将 `DATABASE_READONLY_URL` 设为 Neon PostgreSQL 连接串。Neon 主机会自动使用无服务器 HTTP 驱动；也可以显式设置 `DATABASE_DRIVER=neon`。
 
-在 `.env` 中将 `DATABASE_READONLY_URL` 设为 Neon PostgreSQL 连接串。详见 [`.env.example`](.env.example)。
+```bash
+cp .env.example .env
+cat >> .env <<'EOF'
+DATABASE_READONLY_URL=postgresql://user:password@ep-example.us-east-1.aws.neon.tech/database
+DATABASE_DRIVER=neon
+DATABASE_SSL=true
+EOF
+```
 
 ### 3. 启动开发服务器
 

package.json

@@ -33,8 +33,6 @@
     "admin:db:ingest:ci": "pnpm --filter *db db:ingest:ci",
     "admin:db:ingest:gcs": "pnpm --filter *db db:ingest:gcs",
     "admin:db:ingest:supplemental": "pnpm --filter *db db:ingest:supplemental",
-    "admin:db:dump": "pnpm --filter *db db:dump",
-    "admin:db:load-dump": "pnpm --filter *db db:load-dump --",
     "admin:db:migrate": "pnpm --filter *db db:migrate",
     "admin:db:apply-overrides": "pnpm --filter *db db:apply-overrides",
     "admin:db:reset": "pnpm --filter *db db:reset",

packages/app/src/app/api/v1/agentic-aggregates/route.ts

@@ -1,5 +1,5 @@
-import { JSON_MODE, getDb } from '@semianalysisai/inferencex-db/connection';
-import * as jsonProvider from '@semianalysisai/inferencex-db/json-provider';
+import { getDb } from '@semianalysisai/inferencex-db/connection';
+
 import {
   getAgenticAggregates,
   STATS_VERSION,
@@ -24,10 +24,7 @@ export const dynamic = 'force-dynamic';
 export const CACHE_KEY_PREFIX = `agentic-aggregates-v${STATS_VERSION}`;
 
 const getCachedAgenticAggregates = cachedQuery(
-  (ids: number[]): Promise<AgenticAggregateMap> => {
-    if (JSON_MODE) return Promise.resolve(jsonProvider.getAgenticAggregates(ids));
-    return getAgenticAggregates(getDb(), ids);
-  },
+  (ids: number[]): Promise<AgenticAggregateMap> => getAgenticAggregates(getDb(), ids),
   CACHE_KEY_PREFIX,
   { blobOnly: true },
 );

packages/app/src/app/api/v1/agentic-cache-keys.test.ts

@@ -14,7 +14,6 @@ import { describe, expect, it, vi } from 'vitest';
 // in the blob cache — stub both so importing the route is side-effect-free.
 vi.mock('@semianalysisai/inferencex-db/connection', () => ({
   getDb: vi.fn(() => 'mock-sql'),
-  JSON_MODE: false,
   FIXTURES_MODE: false,
 }));
 

packages/app/src/app/api/v1/availability/route.test.ts

@@ -7,7 +7,6 @@ const { mockGetAvailabilityData, mockGetDb } = vi.hoisted(() => ({
 
 vi.mock('@semianalysisai/inferencex-db/connection', () => ({
   getDb: mockGetDb,
-  JSON_MODE: false,
   FIXTURES_MODE: false,
 }));
 

packages/app/src/app/api/v1/availability/route.ts

@@ -1,18 +1,15 @@
 import { NextResponse } from 'next/server';
 
-import { FIXTURES_MODE, JSON_MODE, getDb } from '@semianalysisai/inferencex-db/connection';
-import * as jsonProvider from '@semianalysisai/inferencex-db/json-provider';
+import { FIXTURES_MODE, getDb } from '@semianalysisai/inferencex-db/connection';
+
 import { getAvailabilityData } from '@semianalysisai/inferencex-db/queries/workflow-info';
 
 import { cachedJson, cachedQuery } from '@/lib/api-cache';
 import { loadFixture } from '@/lib/test-fixtures';
 
 export const dynamic = 'force-dynamic';
 
-const getCachedAvailability = cachedQuery(() => {
-  if (JSON_MODE) return Promise.resolve(jsonProvider.getAvailabilityData());
-  return getAvailabilityData(getDb());
-}, 'availability');
+const getCachedAvailability = cachedQuery(() => getAvailabilityData(getDb()), 'availability');
 
 export async function GET() {
   if (FIXTURES_MODE) return cachedJson(loadFixture('availability'));

packages/app/src/app/api/v1/benchmark-siblings/route.ts

@@ -1,5 +1,5 @@
-import { JSON_MODE, getDb } from '@semianalysisai/inferencex-db/connection';
-import * as jsonProvider from '@semianalysisai/inferencex-db/json-provider';
+import { getDb } from '@semianalysisai/inferencex-db/connection';
+
 import {
   getBenchmarkSiblings,
   type BenchmarkSiblings,
@@ -11,10 +11,10 @@ import { idQueryRoute } from '../id-routes';
 
 export const dynamic = 'force-dynamic';
 
-const getCachedSiblings = cachedQuery((id: number): Promise<BenchmarkSiblings | null> => {
-  if (JSON_MODE) return Promise.resolve(jsonProvider.getBenchmarkSiblings(id));
-  return getBenchmarkSiblings(getDb(), id);
-}, 'benchmark-siblings');
+const getCachedSiblings = cachedQuery(
+  (id: number): Promise<BenchmarkSiblings | null> => getBenchmarkSiblings(getDb(), id),
+  'benchmark-siblings',
+);
 
 /**
  * GET /api/v1/benchmark-siblings?id=N