fix: materialize plugin-declared storage indexes

.changeset/plugin-storage-indexes.md

@@ -0,0 +1,5 @@
+---
+"emdash": patch
+---
+
+Fixes plugin-declared storage indexes never being created. Indexes declared in a plugin manifest's `storage` section are now materialized on marketplace/registry install and update, dropped on uninstall, and synced for configured plugins on the scheduler tick — so plugin storage queries (like the audit-log dashboard widgets) use an index instead of scanning the whole table. The index shape also changed to a composite that SQLite actually uses on D1, where table statistics are never collected.

packages/core/src/api/handlers/marketplace.ts

@@ -24,6 +24,10 @@ import {
 } from "../../plugins/marketplace.js";
 import type { SandboxRunner } from "../../plugins/sandbox/types.js";
 import { PluginStateRepository } from "../../plugins/state.js";
+import {
+	removeAllPluginIndexes,
+	syncDeclaredStorageIndexes,
+} from "../../plugins/storage-indexes.js";
 import { normalizeCapabilities } from "../../plugins/types.js";
 import type { PluginManifest } from "../../plugins/types.js";
 import { EmDashStorageError } from "../../storage/types.js";
@@ -475,6 +479,8 @@ export async function handleMarketplaceInstall(
 			description: pluginDetail.description ?? undefined,
 		});
 
+		await syncDeclaredStorageIndexes(db, [bundle.manifest]);
+
 		// Fire-and-forget install stat
 		client.reportInstall(pluginId, version).catch(() => {
 			// Intentional: never fails the install
@@ -713,6 +719,8 @@ export async function handleMarketplaceUpdate(
 			mcpToolsConsent: null,
 		});
 
+		await syncDeclaredStorageIndexes(db, [bundle.manifest]);
+
 		// Clean up old bundle from R2 (best-effort)
 		deleteBundleFromR2(storage, pluginId, oldVersion).catch(() => {});
 
@@ -786,6 +794,12 @@ export async function handleMarketplaceUninstall(
 			}
 		}
 
+		try {
+			await removeAllPluginIndexes(db, pluginId);
+		} catch {
+			// Nothing to drop, or tracking table predates the feature
+		}
+
 		// Delete state row
 		await stateRepo.delete(pluginId);
 

packages/core/src/api/handlers/registry.ts

@@ -49,6 +49,10 @@ import { extractBundle } from "../../plugins/marketplace.js";
 import type { PluginBundle } from "../../plugins/marketplace.js";
 import type { SandboxRunner } from "../../plugins/sandbox/types.js";
 import { PluginStateRepository } from "../../plugins/state.js";
+import {
+	removeAllPluginIndexes,
+	syncDeclaredStorageIndexes,
+} from "../../plugins/storage-indexes.js";
 import { declaredAccessToCapabilities } from "../../plugins/types.js";
 import type { DeclaredAccess } from "../../plugins/types.js";
 import {
@@ -1181,6 +1185,8 @@ export async function handleRegistryInstall(
 			throw stateErr;
 		}
 
+		await syncDeclaredStorageIndexes(db, [bundle.manifest]);
+
 		return {
 			success: true,
 			data: {
@@ -1288,6 +1294,12 @@ export async function handleRegistryUninstall(
 			await deleteBundleFromR2(storage, pluginId, version, "registry");
 		}
 
+		try {
+			await removeAllPluginIndexes(db, pluginId);
+		} catch {
+			// Nothing to drop, or tracking table predates the feature
+		}
+
 		await stateRepo.delete(pluginId);
 
 		return { success: true, data: { pluginId, dataDeleted } };
@@ -1646,6 +1658,8 @@ export async function handleRegistryUpdate(
 			mcpToolsConsent: null,
 		});
 
+		await syncDeclaredStorageIndexes(db, [bundle.manifest]);
+
 		// Best-effort cleanup of the old bundle. Failures here don't roll
 		// back the upgrade (the new bundle is already stored and committed
 		// in the state row); the orphan is just storage we'll pay for.

packages/core/src/database/validate.ts

@@ -26,6 +26,13 @@ const GENERIC_IDENTIFIER_PATTERN = /^[a-zA-Z][a-zA-Z0-9_]*$/;
  */
 const PLUGIN_IDENTIFIER_PATTERN = /^[a-z][a-z0-9_-]*$/;
 
+/**
+ * Pattern for plugin storage collection names.
+ * Manifests declare these as free-form keys, so both cases and hyphens are
+ * allowed; the charset still excludes quotes, whitespace, and punctuation.
+ */
+const STORAGE_COLLECTION_PATTERN = /^[a-zA-Z][a-zA-Z0-9_-]*$/;
+
 /**
  * Maximum length for SQL identifiers.
  * SQLite has no formal limit, but we cap at 128 for sanity.
@@ -136,3 +143,35 @@ export function validatePluginIdentifier(value: string, label = "plugin identifi
 		throw new IdentifierError(`${label} must match /^[a-z][a-z0-9_-]*$/ (got "${value}")`, value);
 	}
 }
+
+/**
+ * Validate a plugin storage collection name.
+ *
+ * Collections are declared as free-form manifest keys and stored as opaque
+ * text, so this is deliberately more permissive than `validateIdentifier`:
+ * `form-submissions` and `formSubmissions` are legitimate. The charset still
+ * rejects quotes and punctuation, keeping generated index names inert.
+ *
+ * @param value - The string to validate
+ * @param label - Human-readable label for error messages
+ * @throws {IdentifierError} If the value is not valid
+ */
+export function validateStorageCollectionName(value: string, label = "collection name"): void {
+	if (!value || typeof value !== "string") {
+		throw new IdentifierError(`${label} must be a non-empty string`, String(value));
+	}
+
+	if (value.length > MAX_IDENTIFIER_LENGTH) {
+		throw new IdentifierError(
+			`${label} must be ${MAX_IDENTIFIER_LENGTH} characters or less, got ${value.length}`,
+			value,
+		);
+	}
+
+	if (!STORAGE_COLLECTION_PATTERN.test(value)) {
+		throw new IdentifierError(
+			`${label} must match /^[a-zA-Z][a-zA-Z0-9_-]*$/ (got "${value}")`,
+			value,
+		);
+	}
+}

packages/core/src/emdash-runtime.ts

@@ -196,6 +196,7 @@ import { extractRequestMeta, sanitizeHeadersForSandbox } from "./plugins/request
 import { buildRouteMeta, PluginRouteRegistry, type RouteMeta } from "./plugins/routes.js";
 import type { CronScheduler } from "./plugins/scheduler/types.js";
 import { PluginStateRepository } from "./plugins/state.js";
+import { syncDeclaredStorageIndexes } from "./plugins/storage-indexes.js";
 import { normalizeRegistryConfig } from "./registry/config.js";
 import { requestCached } from "./request-cache.js";
 import { getRequestContext } from "./request-context.js";
@@ -492,6 +493,7 @@ const marketplaceManifestCache = new Map<
 			settingsSchema?: Record<string, SettingField>;
 		};
 		mcp?: PluginMcpManifestConfig;
+		storage?: PluginManifest["storage"];
 	}
 >();
 /** Route metadata for sandboxed plugins: pluginId -> routeName -> RouteMeta */
@@ -553,6 +555,8 @@ export class EmDashRuntime {
 	/** All plugins eligible for the hook pipeline (includes built-in plugins).
 	 *  Stored so we can rebuild the pipeline when plugins are enabled/disabled. */
 	private allPipelinePlugins: ResolvedPlugin[];
+	/** Guards the once-per-process plugin storage-index sync. */
+	private storageIndexesSynced = false;
 	/** Factory options for the hook pipeline context factory */
 	private pipelineFactoryOptions: {
 		db: Kysely<Database>;
@@ -683,12 +687,36 @@ export class EmDashRuntime {
 			console.error("[cleanup] System cleanup failed:", error);
 		}
 
+		try {
+			await this.syncPluginStorageIndexesOnce();
+		} catch (error) {
+			console.error("[plugins] Storage index sync failed:", error);
+		}
+
 		// Never throws; no-op unless scheduled backups are enabled and due.
 		await maybeRunScheduledBackup(this.db, this.storage ?? undefined);
 
 		return { published };
 	}
 
+	/**
+	 * Materialize plugin-declared storage indexes, once per process.
+	 *
+	 * Called from the scheduler path, not from request handlers — configured
+	 * plugins have no install handler, so the tick is their only sync moment.
+	 */
+	async syncPluginStorageIndexesOnce(): Promise<void> {
+		if (this.storageIndexesSynced) return;
+		this.storageIndexesSynced = true;
+		// Sandboxed marketplace/registry plugins never join allPipelinePlugins;
+		// their manifests are cached at bundle load. Without them, plugins
+		// installed before this feature shipped would never get their indexes.
+		await syncDeclaredStorageIndexes(this.db, [
+			...this.allPipelinePlugins,
+			...marketplaceManifestCache.values(),
+		]);
+	}
+
 	/**
 	 * Stop the cron scheduler gracefully.
 	 * Call during worker shutdown or hot-reload.
@@ -896,6 +924,7 @@ export class EmDashRuntime {
 					version: bundle.manifest.version,
 					admin: bundle.manifest.admin,
 					mcp: bundle.manifest.mcp,
+					storage: bundle.manifest.storage,
 				});
 
 				// Cache route metadata from manifest for auth decisions
@@ -1011,6 +1040,7 @@ export class EmDashRuntime {
 					version: bundle.manifest.version,
 					admin: bundle.manifest.admin,
 					mcp: bundle.manifest.mcp,
+					storage: bundle.manifest.storage,
 				});
 				if (bundle.manifest.routes.length > 0) {
 					const routeMetaMap = new Map<string, RouteMeta>();
@@ -1612,6 +1642,11 @@ export class EmDashRuntime {
 							// by runSystemCleanup. This catches unexpected errors.
 							console.error("[cleanup] System cleanup failed:", error);
 						}
+						try {
+							await runtimeRef.current?.syncPluginStorageIndexesOnce();
+						} catch (error) {
+							console.error("[plugins] Storage index sync failed:", error);
+						}
 						// Never throws; no-op unless scheduled backups are enabled and due.
 						await maybeRunScheduledBackup(db, storage ?? undefined);
 					});
@@ -2090,6 +2125,7 @@ export class EmDashRuntime {
 						version: bundle.manifest.version,
 						admin: bundle.manifest.admin,
 						mcp: bundle.manifest.mcp,
+						storage: bundle.manifest.storage,
 					});
 
 					// Cache route metadata from manifest for auth decisions
@@ -2162,6 +2198,7 @@ export class EmDashRuntime {
 						version: bundle.manifest.version,
 						admin: bundle.manifest.admin,
 						mcp: bundle.manifest.mcp,
+						storage: bundle.manifest.storage,
 					});
 					if (bundle.manifest.routes.length > 0) {
 						const routeMeta = new Map<string, RouteMeta>();

packages/core/src/plugins/storage-indexes.ts

@@ -12,9 +12,9 @@ import { sql } from "kysely";
 import { jsonExtractExpr, isPostgres } from "../database/dialect-helpers.js";
 import type { Database } from "../database/types.js";
 import {
-	validateIdentifier,
 	validateJsonFieldName,
 	validatePluginIdentifier,
+	validateStorageCollectionName,
 } from "../database/validate.js";
 
 /**
@@ -36,8 +36,10 @@ export function generateIndexName(
 /**
  * Generate a Kysely sql expression for creating an expression index.
  *
- * Validates all identifiers before interpolation to prevent SQL injection.
- * Plugin ID and collection values are parameterized in the WHERE clause.
+ * Validates all inputs before interpolation. The collection uses the
+ * permissive manifest-key rules rather than SQL-identifier rules — it is
+ * stored as opaque text and only reaches SQL inside the generated index
+ * name — so kebab-case collections index like any other.
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
 export function generateCreateIndexSql(
@@ -47,9 +49,8 @@ export function generateCreateIndexSql(
 	fields: string[],
 	options?: { unique?: boolean },
 ): RawBuilder<unknown> {
-	// Validate all identifiers
 	validatePluginIdentifier(pluginId, "plugin ID");
-	validateIdentifier(collection, "collection name");
+	validateStorageCollectionName(collection, "collection name");
 	for (const field of fields) {
 		validateJsonFieldName(field, "index field name");
 	}
@@ -68,14 +69,15 @@ export function generateCreateIndexSql(
 		})
 		.join(", ");
 
-	// Partial index filtered to this plugin/collection
-	// SQLite prohibits bound parameters in partial index WHERE clauses,
-	// so we use sql.lit() for literal string values. Both pluginId and
-	// collection are validated above, so this is safe.
+	// Composite non-partial index: the leading (plugin_id, collection) columns
+	// scope it per plugin/collection — including unique-index semantics — and
+	// let it serve the repository's bound-parameter WHERE plus the JSON
+	// expression ORDER BY. A partial index (WHERE plugin_id = 'x' AND
+	// collection = 'y') is never chosen by SQLite under bound parameters
+	// unless ANALYZE has run, and D1 never runs ANALYZE.
 	const createKeyword = options?.unique ? "CREATE UNIQUE INDEX" : "CREATE INDEX";
 	return sql`${sql.raw(createKeyword)} IF NOT EXISTS ${sql.ref(indexName)}
-		ON _plugin_storage(${sql.raw(expressions)})
-		WHERE plugin_id = ${sql.lit(pluginId)} AND collection = ${sql.lit(collection)}
+		ON _plugin_storage(plugin_id, collection, ${sql.raw(expressions)})
 	`;
 }
 
@@ -254,6 +256,43 @@ export async function syncStorageIndexes(
 	};
 }
 
+/**
+ * Materialize the storage indexes a set of plugins declare in their
+ * manifests. Failures are logged per collection and never thrown — a missing
+ * index affects query performance, not correctness, so it must not fail an
+ * install or a scheduler tick.
+ */
+export async function syncDeclaredStorageIndexes(
+	db: Kysely<Database>,
+	plugins: Array<{
+		id: string;
+		storage?: Record<
+			string,
+			{ indexes: Array<string | string[]>; uniqueIndexes?: Array<string | string[]> }
+		>;
+	}>,
+): Promise<void> {
+	for (const plugin of plugins) {
+		for (const [collection, config] of Object.entries(plugin.storage ?? {})) {
+			try {
+				const result = await syncStorageIndexes(db, plugin.id, collection, config.indexes, {
+					uniqueIndexes: config.uniqueIndexes,
+				});
+				for (const failure of result.errors) {
+					console.error(
+						`[plugins] Failed to sync storage index ${failure.index} for ${plugin.id}/${collection}: ${failure.error}`,
+					);
+				}
+			} catch (error) {
+				console.error(
+					`[plugins] Failed to sync storage indexes for ${plugin.id}/${collection}:`,
+					error,
+				);
+			}
+		}
+	}
+}
+
 /**
  * Remove all indexes for a plugin
  */