fix: replace better-sqlite3 with bun:sqlite for schema cache and SQLite driver (#323)

* fix: replace `better-sqlite3` with `bun:sqlite` for schema cache and SQLite driver

`better-sqlite3` is a native addon that doesn't work on Bun ("not yet supported").
This broke `schema_index`, `schema_search`, `schema_cache_status`, and the SQLite
driver for all users on the released CLI binary.

Switch to `bun:sqlite` which is built into the Bun runtime — zero-install, no
native compilation, same synchronous API. The storage layer (`db.ts`) already
uses this pattern successfully.

Changes:
- `schema/cache.ts`: direct `bun:sqlite` import, sync `create()`/`createInMemory()`
- `drivers/sqlite.ts`: `bun:sqlite` import, fix PRAGMA+LIMIT syntax error
- Remove `better-sqlite3` from optional deps, peer deps, build externals, types
- Update driver docs and E2E tests (SQLite tests no longer need skip guards)

Closes #314

* test: add adversarial schema cache and SQLite driver tests

34 new tests covering:
- Upgrade path from `better-sqlite3` (legacy DB files opened by `bun:sqlite`)
- Corrupted/truncated/zero-byte database files
- SQL injection resistance via search queries (7 injection vectors + null bytes)
- Unicode and special character identifiers in schema/table/column names
- Large dataset stress (1000 tables x 10 columns)
- Re-indexing data replacement and multi-warehouse isolation
- Connector failure modes (listSchemas/listTables/describeTable errors)
- Search edge cases (empty, stop words, long query, case insensitivity, FQN)
- `listColumns` with limits and unknown warehouses
- File-based cache persistence across close/reopen
- Singleton lifecycle (getCache/resetCache)
- PRAGMA LIMIT syntax fix verification

* fix: address code review findings — readonly bug, transaction wrapping, tests

Fixes from 6-model consensus code review:

1. **CRITICAL** — SQLite driver `readonly` + `create: true` bug: gate `create`
   and WAL pragma on `!readonly` so readonly connections don't silently open
   read-write or crash on PRAGMA WAL.

2. **MAJOR** — Wrap `indexWarehouse` inserts in `db.transaction()` per-table
   to avoid per-statement disk fsyncs (~200x slowdown for large warehouses).

3. **MAJOR** — Fix no-op parent directory test (was creating dir before testing).
   Add 3 readonly connection tests (read existing, reject writes, refuse create).

4. **MINOR** — Extend `idx_columns_table` covering index to include
   `column_name` for `listColumns()` ORDER BY.

Author: anandgupta42

docs/docs/drivers.md

@@ -10,7 +10,7 @@ Altimate Code connects to 10 databases natively via TypeScript drivers. No Pytho
 |----------|---------|-------------|------------|-------|
 | PostgreSQL | `pg` | Password, Connection String, SSL | ✅ Docker | Stable, fully parameterized queries |
 | DuckDB | `duckdb` | File/Memory (no auth) | ✅ In-memory | Default local database |
-| SQLite | `better-sqlite3` | File (no auth) | ✅ File-based | Sync API wrapped async |
+| SQLite | `bun:sqlite` (built-in) | File (no auth) | ✅ File-based | Zero-install, built into runtime |
 | MySQL | `mysql2` | Password | ✅ Docker | Parameterized introspection |
 | SQL Server | `mssql` | Password, Azure AD | ✅ Docker | Uses `tedious` TDS protocol |
 | Redshift | `pg` (wire-compat) | Password | ✅ Docker (PG wire) | Uses SVV system views |
@@ -26,7 +26,7 @@ Drivers are `optionalDependencies`, so install only what you need:
 ```bash
 # Embedded databases (no external service needed)
 bun add duckdb
-bun add better-sqlite3
+# SQLite uses bun:sqlite (built-in, no install needed)
 
 # Standard databases
 bun add pg                        # PostgreSQL + Redshift

package.json

@@ -73,7 +73,6 @@
   },
   "devDependencies": {
     "@tsconfig/bun": "catalog:",
-    "@types/better-sqlite3": "7.6.13",
     "@types/pg": "8.18.0",
     "@typescript/native-preview": "catalog:",
     "husky": "9.1.7",

packages/drivers/package.json

@@ -16,7 +16,6 @@
     "mysql2": "^3.0.0",
     "mssql": "^11.0.0",
     "oracledb": "^6.0.0",
-    "duckdb": "^1.0.0",
-    "better-sqlite3": "^11.0.0"
+    "duckdb": "^1.0.0"
   }
 }

packages/drivers/src/sqlite.ts

@@ -1,46 +1,45 @@
 /**
- * SQLite driver using the `better-sqlite3` package.
+ * SQLite driver using Bun's built-in `bun:sqlite`.
  * Synchronous API wrapped in async interface.
  */
 
+import { Database } from "bun:sqlite"
 import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
 
 export async function connect(config: ConnectionConfig): Promise<Connector> {
-  let Database: any
-  try {
-    const mod = await import("better-sqlite3")
-    Database = mod.default || mod
-  } catch {
-    throw new Error(
-      "SQLite driver not installed. Run: npm install better-sqlite3",
-    )
-  }
-
   const dbPath = (config.path as string) ?? ":memory:"
-  let db: any
+  let db: Database | null = null
 
   return {
     async connect() {
+      const isReadonly = config.readonly === true
       db = new Database(dbPath, {
-        readonly: config.readonly === true,
+        readonly: isReadonly,
+        create: !isReadonly,
       })
-      db.pragma("journal_mode = WAL")
+      if (!isReadonly) {
+        db.exec("PRAGMA journal_mode = WAL")
+      }
     },
 
     async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
+      if (!db) throw new Error("SQLite connection not open")
       const effectiveLimit = limit ?? 1000
 
       // Determine if this is a SELECT-like statement
       const trimmed = sql.trim().toLowerCase()
+      const isPragma = trimmed.startsWith("pragma")
       const isSelect =
         trimmed.startsWith("select") ||
-        trimmed.startsWith("pragma") ||
+        isPragma ||
         trimmed.startsWith("with") ||
         trimmed.startsWith("explain")
 
+      // PRAGMA statements don't support LIMIT clause
       let query = sql
       if (
         isSelect &&
+        !isPragma &&
         effectiveLimit &&
         !/\bLIMIT\b/i.test(sql)
       ) {
@@ -59,7 +58,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
       }
 
       const stmt = db.prepare(query)
-      const rows = stmt.all()
+      const rows = stmt.all() as any[]
       const columns = rows.length > 0 ? Object.keys(rows[0]) : []
       const truncated = rows.length > effectiveLimit
       const limitedRows = truncated ? rows.slice(0, effectiveLimit) : rows
@@ -82,11 +81,12 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
     async listTables(
       _schema: string,
     ): Promise<Array<{ name: string; type: string }>> {
+      if (!db) throw new Error("SQLite connection not open")
       const rows = db
         .prepare(
           "SELECT name, type FROM sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%' ORDER BY name",
         )
-        .all()
+        .all() as any[]
       return rows.map((r: any) => ({
         name: r.name as string,
         type: r.type as string,
@@ -97,7 +97,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
       _schema: string,
       table: string,
     ): Promise<SchemaColumn[]> {
-      const rows = db.prepare('SELECT * FROM pragma_table_info(?) ORDER BY cid').all(table)
+      if (!db) throw new Error("SQLite connection not open")
+      const rows = db.prepare('SELECT * FROM pragma_table_info(?) ORDER BY cid').all(table) as any[]
       return rows.map((r: any) => ({
         name: r.name as string,
         data_type: r.type as string,

packages/opencode/script/build.ts

@@ -242,7 +242,7 @@ for (const item of targets) {
       "@altimateai/altimate-core",
       // Database drivers — native addons, users install on demand per warehouse
       "pg", "snowflake-sdk", "@google-cloud/bigquery", "@databricks/sql",
-      "mysql2", "mssql", "oracledb", "duckdb", "better-sqlite3",
+      "mysql2", "mssql", "oracledb", "duckdb",
       // Optional infra packages — native addons or heavy optional deps
       "keytar", "ssh2", "dockerode",
     ],

packages/opencode/script/publish.ts

@@ -29,7 +29,6 @@ const driverPeerDependencies: Record<string, string> = {
   "mssql": ">=11",
   "oracledb": ">=6",
   "duckdb": ">=1",
-  "better-sqlite3": ">=11",
 }
 
 const driverPeerDependenciesMeta: Record<string, { optional: true }> = Object.fromEntries(

packages/opencode/src/altimate/native/schema/cache.ts

@@ -1,11 +1,12 @@
 /**
  * Schema cache — indexes warehouse metadata into SQLite for fast search.
  *
- * Uses better-sqlite3 (optional dependency, dynamically imported) to build
- * a local FTS-ready cache of warehouse schemas, tables, and columns.
+ * Uses bun:sqlite (built into the Bun runtime) to build a local FTS-ready
+ * cache of warehouse schemas, tables, and columns.
  * Cache location: ~/.altimate-code/schema-cache.db
  */
 
+import { Database } from "bun:sqlite"
 import * as path from "path"
 import * as os from "os"
 import * as fs from "fs"
@@ -65,7 +66,7 @@ CREATE INDEX IF NOT EXISTS idx_tables_search ON tables_cache(search_text);
 CREATE INDEX IF NOT EXISTS idx_columns_search ON columns_cache(search_text);
 CREATE INDEX IF NOT EXISTS idx_tables_warehouse ON tables_cache(warehouse);
 CREATE INDEX IF NOT EXISTS idx_columns_warehouse ON columns_cache(warehouse);
-CREATE INDEX IF NOT EXISTS idx_columns_table ON columns_cache(warehouse, schema_name, table_name);
+CREATE INDEX IF NOT EXISTS idx_columns_table ON columns_cache(warehouse, schema_name, table_name, column_name);
 `
 
 // ---------------------------------------------------------------------------
@@ -115,45 +116,24 @@ function tokenizeQuery(query: string): string[] {
 
 /** SQLite-backed schema metadata cache for fast warehouse search. */
 export class SchemaCache {
-  private db: any // better-sqlite3 Database instance
+  private db: Database
   private dbPath: string
 
-  private constructor(db: any, dbPath: string) {
+  private constructor(db: Database, dbPath: string) {
     this.db = db
     this.dbPath = dbPath
   }
 
-  /**
-   * Create a SchemaCache instance.
-   * Uses dynamic import for better-sqlite3 (optional dependency).
-   */
-  static async create(dbPath?: string): Promise<SchemaCache> {
+  /** Create a SchemaCache instance backed by a file on disk. */
+  static create(dbPath?: string): SchemaCache {
     const resolvedPath = dbPath || defaultCachePath()
-    let Database: any
-    try {
-      const mod = await import("better-sqlite3")
-      Database = mod.default || mod
-    } catch {
-      throw new Error(
-        "better-sqlite3 not installed. Install with: npm install better-sqlite3",
-      )
-    }
-    const db = new Database(resolvedPath)
+    const db = new Database(resolvedPath, { create: true })
     db.exec(CREATE_TABLES_SQL)
     return new SchemaCache(db, resolvedPath)
   }
 
-  /**
-   * Create a SchemaCache with an in-memory database (for testing).
-   */
-  static async createInMemory(): Promise<SchemaCache> {
-    let Database: any
-    try {
-      const mod = await import("better-sqlite3")
-      Database = mod.default || mod
-    } catch {
-      throw new Error("better-sqlite3 not installed.")
-    }
+  /** Create a SchemaCache with an in-memory database (for testing). */
+  static createInMemory(): SchemaCache {
     const db = new Database(":memory:")
     db.exec(CREATE_TABLES_SQL)
     return new SchemaCache(db, ":memory:")
@@ -197,6 +177,18 @@ export class SchemaCache {
        VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
     )
 
+    // Batch inserts per-table inside a transaction to avoid per-statement disk fsyncs.
+    // The async connector calls (listTables, describeTable) run outside the transaction;
+    // only the synchronous SQLite inserts are wrapped.
+    const insertTableBatch = this.db.transaction(
+      (tableArgs: any[], columnArgsBatch: any[][]) => {
+        insertTable.run(...tableArgs)
+        for (const colArgs of columnArgsBatch) {
+          insertColumn.run(...colArgs)
+        }
+      },
+    )
+
     for (const schemaName of schemas) {
       if (schemaName.toUpperCase() === "INFORMATION_SCHEMA") continue
       totalSchemas++
@@ -211,27 +203,32 @@ export class SchemaCache {
       for (const tableInfo of tables) {
         totalTables++
         const searchText = makeSearchText(databaseName, schemaName, tableInfo.name, tableInfo.type)
-        insertTable.run(
-          warehouseName, databaseName, schemaName, tableInfo.name, tableInfo.type, searchText,
-        )
 
         let columns: Array<{ name: string; data_type: string; nullable: boolean }> = []
         try {
           columns = await connector.describeTable(schemaName, tableInfo.name)
         } catch {
-          continue
+          // continue with empty columns
         }
 
+        // Build column insert args
+        const columnArgsBatch: any[][] = []
         for (const col of columns) {
           totalColumns++
           const colSearch = makeSearchText(
             databaseName, schemaName, tableInfo.name, col.name, col.data_type,
           )
-          insertColumn.run(
+          columnArgsBatch.push([
             warehouseName, databaseName, schemaName, tableInfo.name,
             col.name, col.data_type, col.nullable ? 1 : 0, colSearch,
-          )
+          ])
         }
+
+        // Insert table + all its columns in a single transaction
+        insertTableBatch(
+          [warehouseName, databaseName, schemaName, tableInfo.name, tableInfo.type, searchText],
+          columnArgsBatch,
+        )
       }
     }
 
@@ -399,7 +396,7 @@ let _cache: SchemaCache | null = null
 
 export async function getCache(): Promise<SchemaCache> {
   if (!_cache) {
-    _cache = await SchemaCache.create()
+    _cache = SchemaCache.create()
   }
   return _cache
 }