Fix repeated SQLite FTS startup rebuilds (emdash-cms#638)

Author: auggernaut

.changeset/ten-flies-admire.md

@@ -0,0 +1,5 @@
+---
+"emdash": patch
+---
+
+Fixes repeated FTS startup rebuilds on SQLite by verifying indexed row counts against the FTS shadow table.

packages/core/src/search/fts-manager.ts

@@ -113,7 +113,6 @@ export class FTSManager {
 		const contentTable = this.getContentTableName(collectionSlug);
 		const fieldList = searchableFields.join(", ");
 		const newFieldList = searchableFields.map((f) => `NEW.${f}`).join(", ");
-
 		// Insert trigger - only index non-deleted content
 		await sql
 			.raw(`
@@ -355,6 +354,7 @@ export class FTSManager {
 		if (!isSqlite(this.db)) return null;
 		this.validateInputs(collectionSlug);
 		const ftsTable = this.getFtsTableName(collectionSlug);
+		const ftsDocsizeTable = `${ftsTable}_docsize`;
 
 		// Check if table exists
 		if (!(await this.ftsTableExists(collectionSlug))) {
@@ -363,7 +363,7 @@ export class FTSManager {
 
 		// Count indexed rows
 		const result = await sql<{ count: number }>`
-			SELECT COUNT(*) as count FROM "${sql.raw(ftsTable)}"
+			SELECT COUNT(*) as count FROM "${sql.raw(ftsDocsizeTable)}"
 		`.execute(this.db);
 
 		return {
@@ -382,10 +382,19 @@ export class FTSManager {
 		if (!isSqlite(this.db)) return false;
 		this.validateInputs(collectionSlug);
 		const ftsTable = this.getFtsTableName(collectionSlug);
+		const ftsDocsizeTable = `${ftsTable}_docsize`;
 		const contentTable = this.getContentTableName(collectionSlug);
+		const fields = await this.getSearchableFields(collectionSlug);
+		const config = await this.getSearchConfig(collectionSlug);
 
 		if (!(await this.ftsTableExists(collectionSlug))) {
-			return false;
+			if (!config?.enabled || fields.length === 0) {
+				return false;
+			}
+
+			console.warn(`FTS index for "${collectionSlug}" is missing. Rebuilding.`);
+			await this.rebuildIndex(collectionSlug, fields, config.weights);
+			return true;
 		}
 
 		// Check 1: Row count mismatch
@@ -394,8 +403,12 @@ export class FTSManager {
 			WHERE deleted_at IS NULL
 		`.execute(this.db);
 
+		// For external-content FTS tables, COUNT(*) on the virtual table is
+		// answered from the backing content table, including soft-deleted rows.
+		// The docsize shadow table tracks the rows actually present in the
+		// full-text index, which is what we need for repair decisions.
 		const ftsCount = await sql<{ count: number }>`
-			SELECT COUNT(*) as count FROM "${sql.raw(ftsTable)}"
+			SELECT COUNT(*) as count FROM "${sql.raw(ftsDocsizeTable)}"
 		`.execute(this.db);
 
 		const contentRows = contentCount.rows[0]?.count ?? 0;

packages/core/tests/integration/search/fts-repair.test.ts

@@ -0,0 +1,91 @@
+import type { Kysely } from "kysely";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { ContentRepository } from "../../../src/database/repositories/content.js";
+import type { Database } from "../../../src/database/types.js";
+import { SchemaRegistry } from "../../../src/schema/registry.js";
+import { FTSManager } from "../../../src/search/fts-manager.js";
+import { searchWithDb } from "../../../src/search/query.js";
+import { setupTestDatabase, teardownTestDatabase } from "../../utils/test-db.js";
+
+describe("FTS repair", () => {
+	let db: Kysely<Database>;
+	let registry: SchemaRegistry;
+	let repo: ContentRepository;
+	let ftsManager: FTSManager;
+	let gameId: string;
+
+	beforeEach(async () => {
+		db = await setupTestDatabase();
+		registry = new SchemaRegistry(db);
+		repo = new ContentRepository(db);
+		ftsManager = new FTSManager(db);
+
+		await registry.createCollection({
+			slug: "game",
+			label: "Games",
+			labelSingular: "Game",
+			supports: ["search"],
+		});
+		await registry.createField("game", {
+			slug: "title",
+			label: "Title",
+			type: "string",
+			searchable: true,
+		});
+		await registry.createField("game", {
+			slug: "blurb",
+			label: "Blurb",
+			type: "text",
+			searchable: true,
+		});
+
+		const created = await repo.create({
+			type: "game",
+			slug: "trail-of-cthulhu",
+			status: "published",
+			publishedAt: new Date().toISOString(),
+			data: {
+				title: "Trail of Cthulhu",
+				blurb: "Investigative horror in the Cthulhu mythos.",
+			},
+		});
+		gameId = created.id;
+
+		await ftsManager.enableSearch("game");
+	});
+
+	afterEach(async () => {
+		await teardownTestDatabase(db);
+	});
+
+	it("recreates a missing FTS table when search remains enabled", async () => {
+		expect(await ftsManager.ftsTableExists("game")).toBe(true);
+
+		await ftsManager.dropFtsTable("game");
+
+		expect(await ftsManager.ftsTableExists("game")).toBe(false);
+		expect(
+			await searchWithDb(db, "cthulhu", {
+				collections: ["game"],
+				status: "published",
+			}),
+		).toEqual({ items: [] });
+
+		await expect(ftsManager.verifyAndRepairAll()).resolves.toBe(1);
+		expect(await ftsManager.ftsTableExists("game")).toBe(true);
+
+		const repaired = await searchWithDb(db, "cthulhu", {
+			collections: ["game"],
+			status: "published",
+		});
+
+		expect(repaired.items).toHaveLength(1);
+		expect(repaired.items[0]?.slug).toBe("trail-of-cthulhu");
+	});
+
+	it("keeps the FTS index in sync after soft delete", async () => {
+		await expect(repo.delete("game", gameId)).resolves.toBe(true);
+		await expect(ftsManager.verifyAndRepairAll()).resolves.toBe(0);
+	});
+});