feat: add local E2E sanity test harness (#461)

Docker-based "new user simulator" that tests the shipped npm artifact:
- Phase 1: verify-install (9 checks: binary, skills, napi, dbt, git)
- Phase 2: smoke-tests (10 E2E tests via altimate run, parallelized)
- Phase 3: resilience (8 tests: SQLite, WAL, sessions, compaction, config)
- PR-aware test generation (git diff → targeted tests)
- Local CI pipeline (bun run ci → typecheck + tests + markers)
- Machine-aware parallelism (2-6 concurrent based on cores/RAM)

27 tests, all passing in ~2:48 on 20-core machine.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: kulvirgit

package.json

@@ -14,7 +14,12 @@
     "prepare": "husky",
     "random": "echo 'Random script'",
     "hello": "echo 'Hello World!'",
-    "test": "echo 'do not run tests from root' && exit 1"
+    "test": "echo 'do not run tests from root' && exit 1",
+    "ci": "test/sanity/ci-local.sh",
+    "ci:full": "test/sanity/ci-local.sh full",
+    "ci:pr": "test/sanity/ci-local.sh pr",
+    "sanity": "docker compose -f test/sanity/docker-compose.yml up --build --abort-on-container-exit --exit-code-from sanity",
+    "sanity:upgrade": "docker compose -f test/sanity/docker-compose.yml -f test/sanity/docker-compose.upgrade.yml up --build --abort-on-container-exit --exit-code-from sanity"
   },
   "workspaces": {
     "packages": [

test/sanity/.env.example

@@ -0,0 +1,6 @@
+# Required for smoke tests (LLM-dependent)
+# WARNING: Never commit real credentials. Use secrets management for production.
+ANTHROPIC_API_KEY=your-api-key-here
+
+# Optional: Snowflake credentials for cloud warehouse testing
+# ALTIMATE_CODE_CONN_SNOWFLAKE_TEST='{"account":"...","user":"...","password":"...","warehouse":"...","database":"..."}'

test/sanity/Dockerfile

@@ -0,0 +1,52 @@
+FROM oven/bun:1.3.10-debian
+
+# System deps (what a real user would have)
+RUN apt-get update && apt-get install -y \
+    git python3 python3-pip python3-venv curl sqlite3 \
+    && rm -rf /var/lib/apt/lists/*
+
+# dbt in venv (matches real user setup)
+RUN python3 -m venv /opt/dbt && \
+    /opt/dbt/bin/pip install --quiet dbt-core dbt-duckdb dbt-postgres
+ENV PATH="/opt/dbt/bin:$PATH"
+
+# Fresh user, clean HOME — simulates new user
+ENV HOME=/home/testuser
+RUN useradd -m testuser
+USER testuser
+WORKDIR /home/testuser
+
+# Copy the built binary directly (simulates what postinstall does)
+# The real npm publish pipeline rewrites workspace:* deps — we can't use npm pack directly.
+# Instead, copy the built binary + run postinstall manually.
+COPY --chown=testuser packages/opencode/dist/@altimateai/ /home/testuser/.altimate-install/dist/@altimateai/
+COPY --chown=testuser packages/opencode/script/postinstall.mjs /home/testuser/.altimate-install/postinstall.mjs
+COPY --chown=testuser packages/opencode/package.json /home/testuser/.altimate-install/package.json
+COPY --chown=testuser .opencode/skills/ /home/testuser/.altimate-install/skills/
+
+# Install altimate-core native binding (required at runtime)
+RUN cd /home/testuser/.altimate-install && \
+    echo '{"dependencies":{"@altimateai/altimate-core":"latest"}}' > package.json && \
+    bun install && \
+    node -e "require('@altimateai/altimate-core')" 2>/dev/null || { echo "FATAL: altimate-core install failed"; exit 1; }
+
+# Link binary to PATH and copy skills to ~/.altimate/builtin/
+# Detect architecture: use TARGETARCH from docker buildx, fall back to uname
+ARG TARGETARCH
+RUN ARCH="${TARGETARCH:-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/')}" && \
+    mkdir -p /home/testuser/.local/bin && \
+    cp /home/testuser/.altimate-install/dist/@altimateai/altimate-code-linux-${ARCH}/bin/altimate /home/testuser/.local/bin/altimate && \
+    chmod +x /home/testuser/.local/bin/altimate && \
+    mkdir -p /home/testuser/.altimate/builtin && \
+    cp -r /home/testuser/.altimate-install/skills/* /home/testuser/.altimate/builtin/
+ENV PATH="/home/testuser/.local/bin:$PATH"
+ENV NODE_PATH="/home/testuser/.altimate-install/node_modules"
+
+# Copy test scripts
+COPY --chown=testuser test/sanity/ /home/testuser/sanity/
+RUN chmod +x /home/testuser/sanity/run.sh \
+    /home/testuser/sanity/phases/*.sh \
+    /home/testuser/sanity/pr-tests/*.sh \
+    /home/testuser/sanity/lib/*.sh
+
+ENTRYPOINT ["/home/testuser/sanity/run.sh"]

test/sanity/Dockerfile.upgrade

@@ -0,0 +1,44 @@
+FROM oven/bun:1.3.10-debian
+
+RUN apt-get update && apt-get install -y \
+    git python3 python3-pip python3-venv curl sqlite3 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m venv /opt/dbt && \
+    /opt/dbt/bin/pip install --quiet dbt-core dbt-duckdb dbt-postgres
+ENV PATH="/opt/dbt/bin:$PATH"
+
+ENV HOME=/home/testuser
+RUN useradd -m testuser
+USER testuser
+WORKDIR /home/testuser
+
+# Install previous version first
+ARG PRIOR_VERSION=latest
+RUN npm install -g @altimateai/altimate-code@${PRIOR_VERSION}
+
+# Record old version for comparison
+RUN altimate --version > /tmp/old-version.txt 2>/dev/null || echo "unknown" > /tmp/old-version.txt
+
+# Run once to seed DB, config, session data
+RUN git init /tmp/seed-project && \
+    cd /tmp/seed-project && \
+    git config user.name "seed" && \
+    git config user.email "seed@test.local" && \
+    echo '{}' > package.json && \
+    git add -A && git commit -q -m "seed"
+
+# Upgrade to new version
+COPY --chown=testuser dist/package.tgz /tmp/altimate-code.tgz
+RUN npm install -g /tmp/altimate-code.tgz
+
+# Copy test scripts
+COPY --chown=testuser test/sanity/ /home/testuser/sanity/
+RUN chmod +x /home/testuser/sanity/run.sh \
+    /home/testuser/sanity/phases/*.sh \
+    /home/testuser/sanity/pr-tests/*.sh \
+    /home/testuser/sanity/lib/*.sh
+
+ENV OLD_VERSION_FILE=/tmp/old-version.txt
+
+ENTRYPOINT ["/home/testuser/sanity/run.sh", "--upgrade"]

test/sanity/ci-local.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+# Local CI pipeline — ports .github/workflows/ci.yml to run locally
+set -euo pipefail
+
+MODE="${1:-fast}"
+REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+EXIT_CODE=0
+
+# Ensure Docker containers are cleaned up on exit (full/pr modes)
+cleanup_docker() {
+  docker compose -f "$REPO_ROOT/test/sanity/docker-compose.yml" down --volumes --remove-orphans 2>/dev/null || true
+}
+trap cleanup_docker EXIT
+
+echo "========================================"
+echo "  Local CI Pipeline — mode: $MODE"
+echo "  Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+echo "========================================"
+
+run_step() {
+  local name="$1"; shift
+  echo ""
+  echo "--- $name ---"
+  if "$@"; then
+    echo "  >>> $name: PASSED"
+  else
+    echo "  >>> $name: FAILED"
+    EXIT_CODE=1
+  fi
+}
+
+# ── Fast mode (default — what pre-push hook runs) ──────────────
+
+echo ""
+echo "=== Fast CI ==="
+
+run_step "Typecheck" bun turbo typecheck
+
+run_step "Unit Tests (opencode)" bash -c "cd $REPO_ROOT/packages/opencode && bun test --timeout 30000"
+
+run_step "Unit Tests (dbt-tools)" bash -c "cd $REPO_ROOT/packages/dbt-tools && bun run test"
+
+# Marker guard (needs upstream remote)
+if git remote | grep -q upstream; then
+  run_step "Marker Guard" bun run "$REPO_ROOT/script/upstream/analyze.ts" --markers --base origin/main --strict
+else
+  echo ""
+  echo "--- Marker Guard ---"
+  echo "  SKIP: upstream remote not configured"
+fi
+
+# ── Full mode ──────────────────────────────────────────────────
+
+if [ "$MODE" = "--full" ] || [ "$MODE" = "full" ]; then
+  echo ""
+  echo "=== Full CI (Docker) ==="
+
+  # Driver E2E with Docker containers
+  run_step "Docker Services Up" docker compose -f "$REPO_ROOT/test/sanity/docker-compose.yml" up -d postgres mysql mssql redshift
+
+  echo "  Waiting for services to be healthy..."
+  HEALTHY=0
+  for _wait in $(seq 1 30); do
+    HEALTHY=$(docker compose -f "$REPO_ROOT/test/sanity/docker-compose.yml" ps --format json 2>/dev/null | grep -c '"healthy"' || echo "0")
+    if [ "$HEALTHY" -ge 4 ]; then break; fi
+    sleep 2
+  done
+
+  if [ "$HEALTHY" -lt 4 ]; then
+    echo "  >>> Docker Services: FAILED ($HEALTHY/4 healthy after 60s)"
+    EXIT_CODE=1
+  else
+    echo "  >>> Docker Services: $HEALTHY/4 healthy"
+  fi
+
+  # Skip driver tests if services aren't healthy
+  if [ "$HEALTHY" -lt 4 ]; then
+    echo "  SKIP: Driver E2E tests (services not healthy)"
+  else
+
+  run_step "Driver E2E (local)" bash -c "cd $REPO_ROOT/packages/opencode && \
+    TEST_PG_HOST=127.0.0.1 TEST_PG_PORT=15432 TEST_PG_PASSWORD=testpass123 \
+    bun test test/altimate/drivers-e2e.test.ts --timeout 30000"
+
+  run_step "Driver E2E (docker)" bash -c "cd $REPO_ROOT/packages/opencode && \
+    TEST_MYSQL_HOST=127.0.0.1 TEST_MYSQL_PORT=13306 TEST_MYSQL_PASSWORD=testpass123 \
+    TEST_MSSQL_HOST=127.0.0.1 TEST_MSSQL_PORT=11433 TEST_MSSQL_PASSWORD='TestPass123!' \
+    TEST_REDSHIFT_HOST=127.0.0.1 TEST_REDSHIFT_PORT=15439 TEST_REDSHIFT_PASSWORD=testpass123 \
+    bun test test/altimate/drivers-docker-e2e.test.ts --timeout 30000"
+
+  # Full sanity suite in Docker
+  run_step "Sanity Suite (Docker)" docker compose -f "$REPO_ROOT/test/sanity/docker-compose.yml" \
+    up --build --abort-on-container-exit --exit-code-from sanity
+
+  fi  # end healthy gate
+fi
+
+# ── PR mode ────────────────────────────────────────────────────
+
+if [ "$MODE" = "--pr" ] || [ "$MODE" = "pr" ]; then
+  echo ""
+  echo "=== PR-Aware Tests ==="
+
+  run_step "Generate PR tests" bash "$REPO_ROOT/test/sanity/pr-tests/generate.sh" origin/main
+  run_step "Run PR tests" bash "$REPO_ROOT/test/sanity/pr-tests/run-pr-tests.sh"
+fi
+
+# ── Summary ────────────────────────────────────────────────────
+
+echo ""
+echo "========================================"
+if [ $EXIT_CODE -eq 0 ]; then
+  echo "  LOCAL CI: ALL PASSED"
+else
+  echo "  LOCAL CI: SOME STEPS FAILED"
+fi
+echo "========================================"
+
+exit $EXIT_CODE

test/sanity/docker-compose.upgrade.yml

@@ -0,0 +1,8 @@
+# Override for upgrade testing — replaces sanity service with Dockerfile.upgrade
+services:
+  sanity:
+    build:
+      context: ../..
+      dockerfile: test/sanity/Dockerfile.upgrade
+      args:
+        PRIOR_VERSION: ${PRIOR_VERSION:-latest}

test/sanity/docker-compose.yml

@@ -0,0 +1,76 @@
+services:
+  sanity:
+    build:
+      context: ../..
+      dockerfile: test/sanity/Dockerfile
+    environment:
+      - ANTHROPIC_API_KEY
+      - ALTIMATE_CODE_CONN_SNOWFLAKE_TEST
+      - TEST_PG_HOST=postgres
+      - TEST_PG_PORT=5432
+      - TEST_PG_PASSWORD=testpass123
+      - TEST_MYSQL_HOST=mysql
+      - TEST_MYSQL_PORT=3306
+      - TEST_MYSQL_PASSWORD=testpass123
+      - TEST_MSSQL_HOST=mssql
+      - TEST_MSSQL_PORT=1433
+      - TEST_MSSQL_PASSWORD=TestPass123!
+      - TEST_REDSHIFT_HOST=redshift
+      - TEST_REDSHIFT_PORT=5432
+      - TEST_REDSHIFT_PASSWORD=testpass123
+    depends_on:
+      postgres:
+        condition: service_healthy
+      mysql:
+        condition: service_healthy
+      mssql:
+        condition: service_healthy
+      redshift:
+        condition: service_healthy
+
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_PASSWORD: testpass123
+    ports:
+      - "15432:5432"
+    healthcheck:
+      test: pg_isready
+      interval: 5s
+      retries: 10
+
+  mysql:
+    image: mysql:8.0
+    environment:
+      MYSQL_ROOT_PASSWORD: testpass123
+      MYSQL_DATABASE: testdb
+    ports:
+      - "13306:3306"
+    healthcheck:
+      test: mysqladmin ping -h 127.0.0.1
+      interval: 5s
+      retries: 20
+
+  mssql:
+    image: mcr.microsoft.com/azure-sql-edge:latest
+    environment:
+      ACCEPT_EULA: "Y"
+      MSSQL_SA_PASSWORD: "TestPass123!"
+    ports:
+      - "11433:1433"
+    healthcheck:
+      test: /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'TestPass123!' -Q 'SELECT 1' || exit 1
+      interval: 10s
+      retries: 20
+
+  redshift:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_PASSWORD: testpass123
+      POSTGRES_DB: dev
+    ports:
+      - "15439:5432"
+    healthcheck:
+      test: pg_isready
+      interval: 5s
+      retries: 10

test/sanity/fixtures/broken-config.json

@@ -0,0 +1,12 @@
+{
+  "provider": {
+    "anthropic": {
+      "api_key": 12345
+    }
+  },
+  "experimental": {
+    "not_a_real_field": "should be ignored",
+    "auto_mcp_discovery": "not-a-boolean"
+  },
+  "unknown_top_level": true
+}

test/sanity/fixtures/compaction-session.sql

@@ -0,0 +1,7 @@
+-- Seed a large session to test compaction circuit breaker
+-- This creates a session with enough message data to trigger isOverflow()
+-- Note: This is a minimal seed — actual compaction depends on token counting
+-- which requires the LLM provider. This just ensures the DB structure is valid.
+
+INSERT OR IGNORE INTO session (id, project_id, slug, directory, title, version, time_created, time_updated)
+VALUES ('ses_sanity_compaction_test', 'proj_sanity', 'sanity-compaction', '/tmp', 'Compaction Test Session', 1, strftime('%s','now') * 1000, strftime('%s','now') * 1000);

test/sanity/fixtures/old-config.json

@@ -0,0 +1,10 @@
+{
+  "provider": {
+    "anthropic": {
+      "api_key": "test-key-not-real"
+    }
+  },
+  "experimental": {
+    "auto_mcp_discovery": true
+  }
+}