From 994ee0e37be6fa7b47a98482668f0b91f5d8a812 Mon Sep 17 00:00:00 2001 From: Malloo <26630797+MA2153@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:58:42 +0300 Subject: [PATCH 1/5] fix(core): index translation_group with locale so lookups seek MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Translation-group reads filter `translation_group` with `deleted_at IS NULL` and `ORDER BY locale ASC`. Seeking migration 041's `(deleted_at, locale, ...)` composites on `deleted_at` alone already returns rows in locale order, so a stats-blind planner prefers them over the single-column `translation_group` index and reads every non-deleted row in the table. D1 never persists sqlite_stat1, so the index shape is the only available lever. Replace the single-column index with `(translation_group, locale)`, which serves both the equality seek and the sort. Migration 055 creates the replacement before dropping the old index, following 052, because D1 DDL is non-transactional. The batched `translation_group IN (...)` variant does not respond to the same fix: the planner picks the 041 composite there even with no ORDER BY at all. Lead its sort with `translation_group` so the ordering is an exact prefix of the new index. Callers group by translation_group and depend only on locale order within a group, which is unchanged. The dominant caller is the logged-out path — hreflang alternates on public renders and the site query layer — not the admin editor panel. Closes #2316 Co-Authored-By: Claude Opus 5 --- .changeset/translation-group-locale-index.md | 5 + ..._content_translation_group_locale_index.ts | 53 +++++++ .../core/src/database/migrations/runner.ts | 2 + .../core/src/database/repositories/content.ts | 8 +- packages/core/src/schema/registry.ts | 8 +- .../content-translations-plan.test.ts | 136 ++++++++++++++++++ .../integration/database/migrations.test.ts | 1 + .../core/tests/integration/i18n/i18n.test.ts | 2 +- 8 files changed, 211 insertions(+), 4 deletions(-) create mode 100644 .changeset/translation-group-locale-index.md create mode 100644 packages/core/src/database/migrations/055_content_translation_group_locale_index.ts create mode 100644 packages/core/tests/integration/database/content-translations-plan.test.ts diff --git a/.changeset/translation-group-locale-index.md b/.changeset/translation-group-locale-index.md new file mode 100644 index 0000000000..a835d041c1 --- /dev/null +++ b/.changeset/translation-group-locale-index.md @@ -0,0 +1,5 @@ +--- +"emdash": patch +--- + +Fixes translation lookups reading every non-deleted row of a content table. Content tables now carry a `(translation_group, locale)` index, replacing the single-column `translation_group` one, so fetching an entry's translations seeks straight to its group instead of scanning. The improvement is largest on big collections and on D1, where the query planner has no statistics to fall back on. diff --git a/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts b/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts new file mode 100644 index 0000000000..9977442d4e --- /dev/null +++ b/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts @@ -0,0 +1,53 @@ +import type { Kysely } from "kysely"; +import { sql } from "kysely"; + +import { listTablesLike } from "../dialect-helpers.js"; + +/** + * Migration: widen the content `translation_group` index to cover the locale sort. + * + * Translation-group reads filter `translation_group = ?` / `IN (...)` with + * `deleted_at IS NULL` and `ORDER BY locale ASC`. Seeking migration 041's + * `(deleted_at, locale, ...)` composites on `deleted_at` alone already returns + * rows in locale order, so a stats-blind planner prefers them over the + * single-column `translation_group` index and reads every non-deleted row in + * the table. D1 never has `sqlite_stat1`, so the index shape is the only lever. + * + * `(translation_group, locale)` serves both the equality seek and the sort, so + * it wins on the planner's own cost terms without statistics. + * + * Forward-only and idempotent (`IF NOT EXISTS`). + * + * Index names use a short `tg_locale` suffix rather than spelling out + * `translation_group_locale`: Postgres truncates identifiers to 63 bytes, and + * the longer form truncates away the discriminator for long collection slugs. + * Keep this identical to the name in `schema/registry.ts`. + */ +export async function up(db: Kysely): Promise { + const tableNames = await listTablesLike(db, "ec_%"); + + for (const tableName of tableNames) { + // D1 DDL is non-transactional: create the replacement before dropping the + // old index so an interrupted migration always leaves a + // translation_group-leading index in place. + await sql` + CREATE INDEX IF NOT EXISTS ${sql.ref(`idx_${tableName}_tg_locale`)} + ON ${sql.ref(tableName)} (translation_group, locale) + `.execute(db); + + await sql`DROP INDEX IF EXISTS ${sql.ref(`idx_${tableName}_translation_group`)}`.execute(db); + } +} + +export async function down(db: Kysely): Promise { + const tableNames = await listTablesLike(db, "ec_%"); + + for (const tableName of tableNames) { + await sql` + CREATE INDEX IF NOT EXISTS ${sql.ref(`idx_${tableName}_translation_group`)} + ON ${sql.ref(tableName)} (translation_group) + `.execute(db); + + await sql`DROP INDEX IF EXISTS ${sql.ref(`idx_${tableName}_tg_locale`)}`.execute(db); + } +} diff --git a/packages/core/src/database/migrations/runner.ts b/packages/core/src/database/migrations/runner.ts index 5c1c7764ff..33b8eecfde 100644 --- a/packages/core/src/database/migrations/runner.ts +++ b/packages/core/src/database/migrations/runner.ts @@ -57,6 +57,7 @@ import * as m051 from "./051_content_taxonomies_denorm.js"; import * as m052 from "./052_media_usage_read_index.js"; import * as m053 from "./053_plugin_mcp_tools.js"; import * as m054 from "./054_media_upload_attempts.js"; +import * as m055 from "./055_content_translation_group_locale_index.js"; const MIGRATIONS: Readonly> = Object.freeze({ "001_initial": m001, @@ -112,6 +113,7 @@ const MIGRATIONS: Readonly> = Object.freeze({ "052_media_usage_read_index": m052, "053_plugin_mcp_tools": m053, "054_media_upload_attempts": m054, + "055_content_translation_group_locale_index": m055, }); /** Total number of registered migrations. Exported for use in tests. */ diff --git a/packages/core/src/database/repositories/content.ts b/packages/core/src/database/repositories/content.ts index 351a49538c..413167cf84 100644 --- a/packages/core/src/database/repositories/content.ts +++ b/packages/core/src/database/repositories/content.ts @@ -1157,6 +1157,12 @@ export class ContentRepository { * Lets callers resolve many edge groups without an N+1 per group. The caller * groups the flat result by `translationGroup` itself. * + * `translation_group` leads the sort so the ordering is an exact prefix of + * `idx_{table}_tg_locale`. Sorting on `locale` alone makes a stats-blind + * planner scan every non-deleted row via migration 041's `(deleted_at, + * locale, ...)` composites instead. Within a group the locale order callers + * rely on is unchanged. + * * `publishedOnly` restricts the result to `status = 'published'` — reference * reads pass this for callers without `content:read_drafts` so draft/scheduled * entries never leak through an edge traversal. @@ -1183,7 +1189,7 @@ export class ContentRepository { WHERE translation_group IN (${sql.join(chunk)}) AND deleted_at IS NULL ${publishedFilter} - ORDER BY locale ASC + ORDER BY translation_group ASC, locale ASC `.execute(this.db); for (const row of result.rows) items.push(this.mapRow(type, row)); } diff --git a/packages/core/src/schema/registry.ts b/packages/core/src/schema/registry.ts index a2608f45a2..d060f33807 100644 --- a/packages/core/src/schema/registry.ts +++ b/packages/core/src/schema/registry.ts @@ -984,9 +984,13 @@ export class SchemaRegistry { ON ${sql.ref(tableName)} (locale) `.execute(conn); + // Composite so translation-group reads seek here rather than falling into + // the `loc_*` composites below, which a stats-blind planner otherwise + // prefers because `deleted_at` alone already yields locale order (see + // migration 055). Keep this name identical to migration 055. await sql` - CREATE INDEX ${sql.ref(`idx_${tableName}_translation_group`)} - ON ${sql.ref(tableName)} (translation_group) + CREATE INDEX ${sql.ref(`idx_${tableName}_tg_locale`)} + ON ${sql.ref(tableName)} (translation_group, locale) `.execute(conn); // Composite indexes for optimized query performance (see migration 033) diff --git a/packages/core/tests/integration/database/content-translations-plan.test.ts b/packages/core/tests/integration/database/content-translations-plan.test.ts new file mode 100644 index 0000000000..12f64bb43e --- /dev/null +++ b/packages/core/tests/integration/database/content-translations-plan.test.ts @@ -0,0 +1,136 @@ +/** + * Query-plan coverage for translation-group reads on content tables. + * + * SQLite runs without ANALYZE/sqlite_stat1 here, matching D1's stats-blind + * planner. Result parity for these reads is covered by the i18n suite. + */ + +import Database from "better-sqlite3"; +import { Kysely, SqliteDialect } from "kysely"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import * as migration055 from "../../../src/database/migrations/055_content_translation_group_locale_index.js"; +import { runMigrations } from "../../../src/database/migrations/runner.js"; +import { ContentRepository } from "../../../src/database/repositories/content.js"; +import type { Database as DatabaseSchema } from "../../../src/database/types.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; + +interface CapturedQuery { + sql: string; + parameters: readonly unknown[]; +} + +let sqlite: Database.Database; +let db: Kysely; +let repo: ContentRepository; +let captured: CapturedQuery[]; + +beforeEach(async () => { + captured = []; + sqlite = new Database(":memory:"); + db = new Kysely({ + dialect: new SqliteDialect({ database: sqlite }), + log(event) { + if (event.level === "query") { + captured.push({ sql: event.query.sql, parameters: event.query.parameters }); + } + }, + }); + await runMigrations(db); + const registry = new SchemaRegistry(db); + await registry.createCollection({ slug: "post", label: "Posts", labelSingular: "Post" }); + repo = new ContentRepository(db); + + for (let index = 1; index <= 200; index++) { + const group = `tg-${String(Math.ceil(index / 2)).padStart(4, "0")}`; + sqlite + .prepare( + `INSERT INTO ec_post (id, slug, status, locale, translation_group, created_at, updated_at, version) + VALUES (?, ?, 'published', ?, ?, '2025-01-01', '2025-01-01', 1)`, + ) + .run(`id-${index}`, `slug-${index}`, index % 2 === 0 ? "en" : "de", group); + } + captured = []; +}); + +afterEach(async () => { + await db.destroy(); +}); + +it("seeks a single translation group through the translation_group index", async () => { + const items = await repo.findTranslations("post", "tg-0005"); + + expect(items.map((item) => item.locale)).toEqual(["de", "en"]); + + const query = translationQuery(); + const plan = explain(query); + expect(contentAccess(plan)).toMatch( + /SEARCH ec_post USING (?:COVERING )?INDEX idx_ec_post_tg_locale \(translation_group=\?\)/, + ); + expect(plan).not.toContain("SCAN ec_post"); + expect(plan).not.toContain("idx_ec_post_loc_crt"); + expect(plan).not.toContain("idx_ec_post_loc_upd"); +}); + +it("seeks batched translation groups through the translation_group index", async () => { + const items = await repo.findTranslationsForGroups("post", ["tg-0005", "tg-0006"]); + + expect(items).toHaveLength(4); + + const query = translationQuery(); + const plan = explain(query); + expect(contentAccess(plan)).toContain("INDEX idx_ec_post_tg_locale"); + expect(plan).not.toContain("SCAN ec_post"); + expect(plan).not.toContain("idx_ec_post_loc_crt"); + expect(plan).not.toContain("idx_ec_post_loc_upd"); +}); + +it("migrates a pre-055 table off the single-column translation_group index", async () => { + sqlite.exec(`DROP INDEX idx_ec_post_tg_locale`); + sqlite.exec(`CREATE INDEX idx_ec_post_translation_group ON ec_post (translation_group)`); + captured = []; + await repo.findTranslations("post", "tg-0005"); + expect(contentAccess(explain(translationQuery()))).toContain("idx_ec_post_loc_crt"); + + await migration055.up(db); + + expect(indexNames()).not.toContain("idx_ec_post_translation_group"); + captured = []; + await repo.findTranslations("post", "tg-0005"); + expect(contentAccess(explain(translationQuery()))).toMatch( + /SEARCH ec_post USING (?:COVERING )?INDEX idx_ec_post_tg_locale \(translation_group=\?\)/, + ); +}); + +function indexNames(): string[] { + return ( + sqlite.prepare(`SELECT name FROM sqlite_master WHERE type = 'index'`).all() as { + name: string; + }[] + ).map((row) => row.name); +} + +function translationQuery(): CapturedQuery { + const queries = captured.filter((query) => query.sql.includes("translation_group")); + expect(queries).toHaveLength(1); + return queries[0]!; +} + +/** better-sqlite3 only binds primitives; coerce values captured from Kysely. */ +function bindable(parameter: unknown): unknown { + if (typeof parameter === "boolean") return parameter ? 1 : 0; + if (parameter instanceof Date) return parameter.toISOString(); + if (parameter === undefined) return null; + return parameter; +} + +function explain(query: CapturedQuery): string { + const rows = sqlite + .prepare(`EXPLAIN QUERY PLAN ${query.sql}`) + .all(...query.parameters.map(bindable)) as { detail: string }[]; + return rows.map((row) => row.detail).join("\n"); +} + +function contentAccess(plan: string): string | undefined { + return plan.split("\n").find((detail) => /\b(?:SCAN|SEARCH) ec_post\b/.test(detail)); +} diff --git a/packages/core/tests/integration/database/migrations.test.ts b/packages/core/tests/integration/database/migrations.test.ts index ce2e6de4c3..df9af0f17a 100644 --- a/packages/core/tests/integration/database/migrations.test.ts +++ b/packages/core/tests/integration/database/migrations.test.ts @@ -141,6 +141,7 @@ describe("Database Migrations (Integration)", () => { "052_media_usage_read_index", "053_plugin_mcp_tools", "054_media_upload_attempts", + "055_content_translation_group_locale_index", ]; await db.deleteFrom("_emdash_migrations").where("name", "in", trailing).execute(); diff --git a/packages/core/tests/integration/i18n/i18n.test.ts b/packages/core/tests/integration/i18n/i18n.test.ts index da9bc0fe50..df5763eaf7 100644 --- a/packages/core/tests/integration/i18n/i18n.test.ts +++ b/packages/core/tests/integration/i18n/i18n.test.ts @@ -86,7 +86,7 @@ describe("i18n (Integration)", () => { const indexNames = result.rows.map((r) => r.name); expect(indexNames).toContain("idx_ec_post_locale"); - expect(indexNames).toContain("idx_ec_post_translation_group"); + expect(indexNames).toContain("idx_ec_post_tg_locale"); }); }); From c1bcaf4e63504ecbbc93da4ee2a07cc778689a34 Mon Sep 17 00:00:00 2001 From: Malloo <26630797+MA2153@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:16:03 +0300 Subject: [PATCH 2/5] docs(core): trim planner narrative from translation-index comments Review feedback: both comments narrated the rejected alternative (`ORDER BY locale` alone) and cited migration 041 to justify the shape. Keep the invariants a reader needs -- the sort prefix the callers depend on, and the index-name coupling to migration 055 -- and leave the planner rationale in the migration docstring, which is where it belongs. Co-Authored-By: Claude Opus 5 --- packages/core/src/database/repositories/content.ts | 6 ++---- packages/core/src/schema/registry.ts | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/core/src/database/repositories/content.ts b/packages/core/src/database/repositories/content.ts index 413167cf84..e927f94910 100644 --- a/packages/core/src/database/repositories/content.ts +++ b/packages/core/src/database/repositories/content.ts @@ -1158,10 +1158,8 @@ export class ContentRepository { * groups the flat result by `translationGroup` itself. * * `translation_group` leads the sort so the ordering is an exact prefix of - * `idx_{table}_tg_locale`. Sorting on `locale` alone makes a stats-blind - * planner scan every non-deleted row via migration 041's `(deleted_at, - * locale, ...)` composites instead. Within a group the locale order callers - * rely on is unchanged. + * `idx_{table}_tg_locale`; callers group by `translationGroup`, so the + * per-group locale order they rely on is preserved. * * `publishedOnly` restricts the result to `status = 'published'` — reference * reads pass this for callers without `content:read_drafts` so draft/scheduled diff --git a/packages/core/src/schema/registry.ts b/packages/core/src/schema/registry.ts index d060f33807..4269a17999 100644 --- a/packages/core/src/schema/registry.ts +++ b/packages/core/src/schema/registry.ts @@ -984,10 +984,8 @@ export class SchemaRegistry { ON ${sql.ref(tableName)} (locale) `.execute(conn); - // Composite so translation-group reads seek here rather than falling into - // the `loc_*` composites below, which a stats-blind planner otherwise - // prefers because `deleted_at` alone already yields locale order (see - // migration 055). Keep this name identical to migration 055. + // Name must stay identical to migration 055, which creates this index on + // tables that already exist. await sql` CREATE INDEX ${sql.ref(`idx_${tableName}_tg_locale`)} ON ${sql.ref(tableName)} (translation_group, locale) From 0b3a249f735f8eb2b0fcd1ae61387759e278364b Mon Sep 17 00:00:00 2001 From: Malloo <26630797+MA2153@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:31:40 +0300 Subject: [PATCH 3/5] fix(core): keep translation-group reads seeking at full batch size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `(translation_group, locale)` index only held for the single-group read. For the batched variant the `IN (...)` list multiplies the planner's row estimate, so from five groups onward a stats-blind planner fell back to a `deleted_at` composite and read every non-deleted row — the public reference-resolution path batches at SQL_BATCH_SIZE. Leading the index with `deleted_at` matches the read term for term and holds at every batch size, with and without the `publishedOnly` filter. Co-Authored-By: Claude Opus 5 --- .changeset/translation-group-locale-index.md | 2 +- ..._content_translation_group_locale_index.ts | 18 ++++---- .../core/src/database/repositories/content.ts | 6 +-- packages/core/src/schema/registry.ts | 2 +- .../content-translations-plan.test.ts | 41 ++++++++++++------- 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/.changeset/translation-group-locale-index.md b/.changeset/translation-group-locale-index.md index a835d041c1..9c24613486 100644 --- a/.changeset/translation-group-locale-index.md +++ b/.changeset/translation-group-locale-index.md @@ -2,4 +2,4 @@ "emdash": patch --- -Fixes translation lookups reading every non-deleted row of a content table. Content tables now carry a `(translation_group, locale)` index, replacing the single-column `translation_group` one, so fetching an entry's translations seeks straight to its group instead of scanning. The improvement is largest on big collections and on D1, where the query planner has no statistics to fall back on. +Fixes translation lookups reading every non-deleted row of a content table. Content tables now carry a `(deleted_at, translation_group, locale)` index, replacing the single-column `translation_group` one, so fetching an entry's translations — one entry's or a whole page's — seeks straight to the group instead of scanning. The improvement is largest on big collections and on D1, where the query planner has no statistics to fall back on. diff --git a/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts b/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts index 9977442d4e..df63b55048 100644 --- a/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts +++ b/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts @@ -4,17 +4,21 @@ import { sql } from "kysely"; import { listTablesLike } from "../dialect-helpers.js"; /** - * Migration: widen the content `translation_group` index to cover the locale sort. + * Migration: widen the content `translation_group` index to cover the whole + * translation-group read. * - * Translation-group reads filter `translation_group = ?` / `IN (...)` with - * `deleted_at IS NULL` and `ORDER BY locale ASC`. Seeking migration 041's + * Translation-group reads filter `deleted_at IS NULL` with `translation_group = + * ?` / `IN (...)` and `ORDER BY locale ASC`. Seeking migration 041's * `(deleted_at, locale, ...)` composites on `deleted_at` alone already returns * rows in locale order, so a stats-blind planner prefers them over the * single-column `translation_group` index and reads every non-deleted row in - * the table. D1 never has `sqlite_stat1`, so the index shape is the only lever. + * the table. A batched read is worse still: the `IN (...)` list multiplies the + * planner's row estimate for a `translation_group`-leading index, so it falls + * back to a `deleted_at` composite from a handful of groups onward. D1 never + * has `sqlite_stat1`, so the index shape is the only lever. * - * `(translation_group, locale)` serves both the equality seek and the sort, so - * it wins on the planner's own cost terms without statistics. + * `(deleted_at, translation_group, locale)` matches the read term for term, so + * it wins on the planner's own cost terms without statistics at any batch size. * * Forward-only and idempotent (`IF NOT EXISTS`). * @@ -32,7 +36,7 @@ export async function up(db: Kysely): Promise { // translation_group-leading index in place. await sql` CREATE INDEX IF NOT EXISTS ${sql.ref(`idx_${tableName}_tg_locale`)} - ON ${sql.ref(tableName)} (translation_group, locale) + ON ${sql.ref(tableName)} (deleted_at, translation_group, locale) `.execute(db); await sql`DROP INDEX IF EXISTS ${sql.ref(`idx_${tableName}_translation_group`)}`.execute(db); diff --git a/packages/core/src/database/repositories/content.ts b/packages/core/src/database/repositories/content.ts index e927f94910..9f19a10850 100644 --- a/packages/core/src/database/repositories/content.ts +++ b/packages/core/src/database/repositories/content.ts @@ -1157,9 +1157,9 @@ export class ContentRepository { * Lets callers resolve many edge groups without an N+1 per group. The caller * groups the flat result by `translationGroup` itself. * - * `translation_group` leads the sort so the ordering is an exact prefix of - * `idx_{table}_tg_locale`; callers group by `translationGroup`, so the - * per-group locale order they rely on is preserved. + * `translation_group` leads the sort so the ordering follows + * `idx_{table}_tg_locale` past its `deleted_at` equality; callers group by + * `translationGroup`, so the per-group locale order they rely on is preserved. * * `publishedOnly` restricts the result to `status = 'published'` — reference * reads pass this for callers without `content:read_drafts` so draft/scheduled diff --git a/packages/core/src/schema/registry.ts b/packages/core/src/schema/registry.ts index 4269a17999..9c80322b24 100644 --- a/packages/core/src/schema/registry.ts +++ b/packages/core/src/schema/registry.ts @@ -988,7 +988,7 @@ export class SchemaRegistry { // tables that already exist. await sql` CREATE INDEX ${sql.ref(`idx_${tableName}_tg_locale`)} - ON ${sql.ref(tableName)} (translation_group, locale) + ON ${sql.ref(tableName)} (deleted_at, translation_group, locale) `.execute(conn); // Composite indexes for optimized query performance (see migration 033) diff --git a/packages/core/tests/integration/database/content-translations-plan.test.ts b/packages/core/tests/integration/database/content-translations-plan.test.ts index 12f64bb43e..0468faeca8 100644 --- a/packages/core/tests/integration/database/content-translations-plan.test.ts +++ b/packages/core/tests/integration/database/content-translations-plan.test.ts @@ -14,6 +14,7 @@ import { runMigrations } from "../../../src/database/migrations/runner.js"; import { ContentRepository } from "../../../src/database/repositories/content.js"; import type { Database as DatabaseSchema } from "../../../src/database/types.js"; import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { SQL_BATCH_SIZE } from "../../../src/utils/chunks.js"; interface CapturedQuery { sql: string; @@ -65,25 +66,37 @@ it("seeks a single translation group through the translation_group index", async const query = translationQuery(); const plan = explain(query); expect(contentAccess(plan)).toMatch( - /SEARCH ec_post USING (?:COVERING )?INDEX idx_ec_post_tg_locale \(translation_group=\?\)/, + /SEARCH ec_post USING (?:COVERING )?INDEX idx_ec_post_tg_locale \(deleted_at=\? AND translation_group=\?\)/, ); expect(plan).not.toContain("SCAN ec_post"); expect(plan).not.toContain("idx_ec_post_loc_crt"); expect(plan).not.toContain("idx_ec_post_loc_upd"); }); -it("seeks batched translation groups through the translation_group index", async () => { - const items = await repo.findTranslationsForGroups("post", ["tg-0005", "tg-0006"]); - - expect(items).toHaveLength(4); - - const query = translationQuery(); - const plan = explain(query); - expect(contentAccess(plan)).toContain("INDEX idx_ec_post_tg_locale"); - expect(plan).not.toContain("SCAN ec_post"); - expect(plan).not.toContain("idx_ec_post_loc_crt"); - expect(plan).not.toContain("idx_ec_post_loc_upd"); -}); +it.each([ + { groupCount: 2, publishedOnly: false }, + { groupCount: 2, publishedOnly: true }, + { groupCount: SQL_BATCH_SIZE, publishedOnly: false }, + { groupCount: SQL_BATCH_SIZE, publishedOnly: true }, +])( + "seeks $groupCount batched translation groups through the translation_group index (publishedOnly=$publishedOnly)", + async ({ groupCount, publishedOnly }) => { + const groups = Array.from( + { length: groupCount }, + (_, index) => `tg-${String(index + 1).padStart(4, "0")}`, + ); + + const items = await repo.findTranslationsForGroups("post", groups, { publishedOnly }); + + expect(items).toHaveLength(groupCount * 2); + + const plan = explain(translationQuery()); + expect(contentAccess(plan)).toContain("INDEX idx_ec_post_tg_locale"); + expect(plan).not.toContain("SCAN ec_post"); + expect(plan).not.toContain("idx_ec_post_loc_crt"); + expect(plan).not.toContain("idx_ec_post_loc_upd"); + }, +); it("migrates a pre-055 table off the single-column translation_group index", async () => { sqlite.exec(`DROP INDEX idx_ec_post_tg_locale`); @@ -98,7 +111,7 @@ it("migrates a pre-055 table off the single-column translation_group index", asy captured = []; await repo.findTranslations("post", "tg-0005"); expect(contentAccess(explain(translationQuery()))).toMatch( - /SEARCH ec_post USING (?:COVERING )?INDEX idx_ec_post_tg_locale \(translation_group=\?\)/, + /SEARCH ec_post USING (?:COVERING )?INDEX idx_ec_post_tg_locale \(deleted_at=\? AND translation_group=\?\)/, ); }); From a29e012ffff708e854c6d34a8e783eba7c88d268 Mon Sep 17 00:00:00 2001 From: Malloo <26630797+MA2153@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:31:40 +0300 Subject: [PATCH 4/5] test(core): cover the translation index swap for long collection slugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Postgres truncates identifiers to 63 bytes, so a long enough collection slug can collapse migration 055's new and old index names onto the same identifier — the create becomes a no-op and the drop then takes the surviving index with it. Pin the behaviour at the longest slug the registry can create on Postgres. Co-Authored-By: Claude Opus 5 --- .../content-translation-index-name.test.ts | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 packages/core/tests/integration/database/content-translation-index-name.test.ts diff --git a/packages/core/tests/integration/database/content-translation-index-name.test.ts b/packages/core/tests/integration/database/content-translation-index-name.test.ts new file mode 100644 index 0000000000..091df5585c --- /dev/null +++ b/packages/core/tests/integration/database/content-translation-index-name.test.ts @@ -0,0 +1,76 @@ +import { sql } from "kysely"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import * as migration055 from "../../../src/database/migrations/055_content_translation_group_locale_index.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { + type DialectTestContext, + describeEachDialect, + setupForDialect, + teardownForDialect, +} from "../../utils/test-db.js"; + +/** + * The longest collection slug `SchemaRegistry` can create on Postgres: at 47 + * characters the `deleted_updated_id` and `deleted_status` index names collide + * once Postgres truncates identifiers to 63 bytes. `idx_{table}_tg_locale` is + * truncated at this length too, so the migration's create and drop must still + * name two different indexes. + */ +const LONG_SLUG = `t${"o".repeat(45)}`; +const TABLE_NAME = `ec_${LONG_SLUG}`; + +describeEachDialect("translation_group index replacement for long collection slugs", (dialect) => { + let ctx: DialectTestContext; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: LONG_SLUG, label: "Long", labelSingular: "Long" }); + + await sql`DROP INDEX IF EXISTS ${sql.ref(`idx_${TABLE_NAME}_tg_locale`)}`.execute(ctx.db); + await sql` + CREATE INDEX ${sql.ref(`idx_${TABLE_NAME}_translation_group`)} + ON ${sql.ref(TABLE_NAME)} (translation_group) + `.execute(ctx.db); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("leaves the table with the composite index, not with none", async () => { + await migration055.up(ctx.db); + + const covering = (await translationIndexColumns()).filter((columns) => + columns.includes("translation_group"), + ); + expect(covering).toEqual(["deleted_at, translation_group, locale"]); + }); + + async function translationIndexColumns(): Promise { + if (ctx.dialect === "postgres") { + const result = await sql<{ indexdef: string }>` + SELECT indexdef FROM pg_indexes + WHERE schemaname = current_schema() AND tablename = ${TABLE_NAME} + `.execute(ctx.db); + return result.rows.map((row) => columnList(row.indexdef)); + } + + const result = await sql<{ sql: string | null }>` + SELECT sql FROM sqlite_master WHERE type = 'index' AND tbl_name = ${TABLE_NAME} + `.execute(ctx.db); + return result.rows.map((row) => columnList(row.sql ?? "")); + } +}); + +function columnList(definition: string): string { + const open = definition.lastIndexOf("("); + const close = definition.lastIndexOf(")"); + if (open === -1 || close < open) return ""; + return definition + .slice(open + 1, close) + .replaceAll('"', "") + .replaceAll(/\s+/g, " ") + .trim(); +} From b19004ded71dfb31f15f04f4d2e8ad24eca5429f Mon Sep 17 00:00:00 2001 From: Malloo <26630797+MA2153@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:46:46 +0300 Subject: [PATCH 5/5] fix(core): keep a translation_group-leading index for menu lookups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Menu and reference resolution look a content row up by translation_group (+ locale) with no deleted_at term, so the (deleted_at, translation_group, locale) index cannot seek them — the planner fell back to idx_{table}_locale and read every row in the requested locale. Create both shapes: (translation_group, locale) for lookups that don't constrain deleted_at, and (deleted_at, translation_group, locale) for the batched translation-group reads that do. Co-Authored-By: Claude Opus 5 --- .changeset/translation-group-locale-index.md | 2 +- ..._content_translation_group_locale_index.ts | 44 ++++++----- .../core/src/database/repositories/content.ts | 2 +- packages/core/src/schema/registry.ts | 11 ++- .../content-translation-index-name.test.ts | 20 +++-- .../content-translations-plan.test.ts | 73 ++++++++++++++++++- .../core/tests/integration/i18n/i18n.test.ts | 1 + 7 files changed, 120 insertions(+), 33 deletions(-) diff --git a/.changeset/translation-group-locale-index.md b/.changeset/translation-group-locale-index.md index 9c24613486..147db0e076 100644 --- a/.changeset/translation-group-locale-index.md +++ b/.changeset/translation-group-locale-index.md @@ -2,4 +2,4 @@ "emdash": patch --- -Fixes translation lookups reading every non-deleted row of a content table. Content tables now carry a `(deleted_at, translation_group, locale)` index, replacing the single-column `translation_group` one, so fetching an entry's translations — one entry's or a whole page's — seeks straight to the group instead of scanning. The improvement is largest on big collections and on D1, where the query planner has no statistics to fall back on. +Fixes translation lookups reading every non-deleted row of a content table. Content tables now carry `(translation_group, locale)` and `(deleted_at, translation_group, locale)` indexes, replacing the single-column `translation_group` one, so fetching an entry's translations — one entry's, a whole page's, or a menu link's target — seeks straight to the group instead of scanning. The improvement is largest on big collections and on D1, where the query planner has no statistics to fall back on. diff --git a/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts b/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts index df63b55048..2c08648a37 100644 --- a/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts +++ b/packages/core/src/database/migrations/055_content_translation_group_locale_index.ts @@ -4,38 +4,47 @@ import { sql } from "kysely"; import { listTablesLike } from "../dialect-helpers.js"; /** - * Migration: widen the content `translation_group` index to cover the whole - * translation-group read. + * Migration: replace the single-column content `translation_group` index with + * one index per translation-group read shape. * - * Translation-group reads filter `deleted_at IS NULL` with `translation_group = - * ?` / `IN (...)` and `ORDER BY locale ASC`. Seeking migration 041's + * There are two shapes. Translation-group reads filter `deleted_at IS NULL` + * with `translation_group = ?` / `IN (...)` and order by locale; menu and + * reference resolution looks a group up by `translation_group` (+ `locale`) + * alone, with no `deleted_at` term. + * + * Neither shape can borrow the other's index. Seeking migration 041's * `(deleted_at, locale, ...)` composites on `deleted_at` alone already returns - * rows in locale order, so a stats-blind planner prefers them over the + * rows in locale order, so a stats-blind planner prefers them over a * single-column `translation_group` index and reads every non-deleted row in - * the table. A batched read is worse still: the `IN (...)` list multiplies the - * planner's row estimate for a `translation_group`-leading index, so it falls - * back to a `deleted_at` composite from a handful of groups onward. D1 never - * has `sqlite_stat1`, so the index shape is the only lever. - * - * `(deleted_at, translation_group, locale)` matches the read term for term, so - * it wins on the planner's own cost terms without statistics at any batch size. + * the table; the batched variant is worse still, because the `IN (...)` list + * multiplies the planner's row estimate for a `translation_group`-leading index + * and it falls back to a `deleted_at` composite from a handful of groups + * onward. Going the other way, an index leading with `deleted_at` cannot seek a + * lookup that never constrains that column. D1 never has `sqlite_stat1`, so the + * index shape is the only lever and each shape needs a prefix matching it term + * for term. * * Forward-only and idempotent (`IF NOT EXISTS`). * - * Index names use a short `tg_locale` suffix rather than spelling out - * `translation_group_locale`: Postgres truncates identifiers to 63 bytes, and - * the longer form truncates away the discriminator for long collection slugs. - * Keep this identical to the name in `schema/registry.ts`. + * Index names use short `tg_locale` / `del_tg_locale` suffixes rather than + * spelling out `translation_group`: Postgres truncates identifiers to 63 bytes, + * and the longer forms truncate to the same value for long collection slugs. + * Keep these identical to the names in `schema/registry.ts`. */ export async function up(db: Kysely): Promise { const tableNames = await listTablesLike(db, "ec_%"); for (const tableName of tableNames) { - // D1 DDL is non-transactional: create the replacement before dropping the + // D1 DDL is non-transactional: create the replacements before dropping the // old index so an interrupted migration always leaves a // translation_group-leading index in place. await sql` CREATE INDEX IF NOT EXISTS ${sql.ref(`idx_${tableName}_tg_locale`)} + ON ${sql.ref(tableName)} (translation_group, locale) + `.execute(db); + + await sql` + CREATE INDEX IF NOT EXISTS ${sql.ref(`idx_${tableName}_del_tg_locale`)} ON ${sql.ref(tableName)} (deleted_at, translation_group, locale) `.execute(db); @@ -52,6 +61,7 @@ export async function down(db: Kysely): Promise { ON ${sql.ref(tableName)} (translation_group) `.execute(db); + await sql`DROP INDEX IF EXISTS ${sql.ref(`idx_${tableName}_del_tg_locale`)}`.execute(db); await sql`DROP INDEX IF EXISTS ${sql.ref(`idx_${tableName}_tg_locale`)}`.execute(db); } } diff --git a/packages/core/src/database/repositories/content.ts b/packages/core/src/database/repositories/content.ts index 9f19a10850..9a2175558e 100644 --- a/packages/core/src/database/repositories/content.ts +++ b/packages/core/src/database/repositories/content.ts @@ -1158,7 +1158,7 @@ export class ContentRepository { * groups the flat result by `translationGroup` itself. * * `translation_group` leads the sort so the ordering follows - * `idx_{table}_tg_locale` past its `deleted_at` equality; callers group by + * `idx_{table}_del_tg_locale` past its `deleted_at` equality; callers group by * `translationGroup`, so the per-group locale order they rely on is preserved. * * `publishedOnly` restricts the result to `status = 'published'` — reference diff --git a/packages/core/src/schema/registry.ts b/packages/core/src/schema/registry.ts index 9c80322b24..f8f977dd34 100644 --- a/packages/core/src/schema/registry.ts +++ b/packages/core/src/schema/registry.ts @@ -984,10 +984,17 @@ export class SchemaRegistry { ON ${sql.ref(tableName)} (locale) `.execute(conn); - // Name must stay identical to migration 055, which creates this index on - // tables that already exist. + // Names must stay identical to migration 055, which creates these indexes + // on tables that already exist. Lookups that don't constrain `deleted_at` + // (menu and reference resolution) need the first; reads that do need the + // second. await sql` CREATE INDEX ${sql.ref(`idx_${tableName}_tg_locale`)} + ON ${sql.ref(tableName)} (translation_group, locale) + `.execute(conn); + + await sql` + CREATE INDEX ${sql.ref(`idx_${tableName}_del_tg_locale`)} ON ${sql.ref(tableName)} (deleted_at, translation_group, locale) `.execute(conn); diff --git a/packages/core/tests/integration/database/content-translation-index-name.test.ts b/packages/core/tests/integration/database/content-translation-index-name.test.ts index 091df5585c..1ee91f584c 100644 --- a/packages/core/tests/integration/database/content-translation-index-name.test.ts +++ b/packages/core/tests/integration/database/content-translation-index-name.test.ts @@ -13,9 +13,9 @@ import { /** * The longest collection slug `SchemaRegistry` can create on Postgres: at 47 * characters the `deleted_updated_id` and `deleted_status` index names collide - * once Postgres truncates identifiers to 63 bytes. `idx_{table}_tg_locale` is - * truncated at this length too, so the migration's create and drop must still - * name two different indexes. + * once Postgres truncates identifiers to 63 bytes. `idx_{table}_tg_locale` and + * `idx_{table}_del_tg_locale` are truncated at this length too, so the + * migration's creates and drop must still name three different indexes. */ const LONG_SLUG = `t${"o".repeat(45)}`; const TABLE_NAME = `ec_${LONG_SLUG}`; @@ -29,6 +29,7 @@ describeEachDialect("translation_group index replacement for long collection slu await registry.createCollection({ slug: LONG_SLUG, label: "Long", labelSingular: "Long" }); await sql`DROP INDEX IF EXISTS ${sql.ref(`idx_${TABLE_NAME}_tg_locale`)}`.execute(ctx.db); + await sql`DROP INDEX IF EXISTS ${sql.ref(`idx_${TABLE_NAME}_del_tg_locale`)}`.execute(ctx.db); await sql` CREATE INDEX ${sql.ref(`idx_${TABLE_NAME}_translation_group`)} ON ${sql.ref(TABLE_NAME)} (translation_group) @@ -39,13 +40,16 @@ describeEachDialect("translation_group index replacement for long collection slu await teardownForDialect(ctx); }); - it("leaves the table with the composite index, not with none", async () => { + it("leaves the table with both composite indexes, not with one or none", async () => { await migration055.up(ctx.db); - const covering = (await translationIndexColumns()).filter((columns) => - columns.includes("translation_group"), - ); - expect(covering).toEqual(["deleted_at, translation_group, locale"]); + const covering = (await translationIndexColumns()) + .filter((columns) => columns.includes("translation_group")) + .toSorted(); + expect(covering).toEqual([ + "deleted_at, translation_group, locale", + "translation_group, locale", + ]); }); async function translationIndexColumns(): Promise { diff --git a/packages/core/tests/integration/database/content-translations-plan.test.ts b/packages/core/tests/integration/database/content-translations-plan.test.ts index 0468faeca8..1e2b0ff163 100644 --- a/packages/core/tests/integration/database/content-translations-plan.test.ts +++ b/packages/core/tests/integration/database/content-translations-plan.test.ts @@ -13,6 +13,7 @@ import * as migration055 from "../../../src/database/migrations/055_content_tran import { runMigrations } from "../../../src/database/migrations/runner.js"; import { ContentRepository } from "../../../src/database/repositories/content.js"; import type { Database as DatabaseSchema } from "../../../src/database/types.js"; +import { getMenuWithDb } from "../../../src/menus/index.js"; import { SchemaRegistry } from "../../../src/schema/registry.js"; import { SQL_BATCH_SIZE } from "../../../src/utils/chunks.js"; @@ -66,7 +67,7 @@ it("seeks a single translation group through the translation_group index", async const query = translationQuery(); const plan = explain(query); expect(contentAccess(plan)).toMatch( - /SEARCH ec_post USING (?:COVERING )?INDEX idx_ec_post_tg_locale \(deleted_at=\? AND translation_group=\?\)/, + /SEARCH ec_post USING (?:COVERING )?INDEX idx_ec_post_del_tg_locale \(deleted_at=\? AND translation_group=\?\)/, ); expect(plan).not.toContain("SCAN ec_post"); expect(plan).not.toContain("idx_ec_post_loc_crt"); @@ -91,15 +92,42 @@ it.each([ expect(items).toHaveLength(groupCount * 2); const plan = explain(translationQuery()); - expect(contentAccess(plan)).toContain("INDEX idx_ec_post_tg_locale"); + expect(contentAccess(plan)).toContain("INDEX idx_ec_post_del_tg_locale"); expect(plan).not.toContain("SCAN ec_post"); expect(plan).not.toContain("idx_ec_post_loc_crt"); expect(plan).not.toContain("idx_ec_post_loc_upd"); }, ); +/** + * Menu references resolve a translation group without a `deleted_at` filter, so + * they need an index that leads with `translation_group`. `fr` has no `ec_post` + * row, which exercises the any-locale fallback as well as the direct lookup. + */ +it.each([ + { locale: "en", url: "/post/slug-10" }, + { locale: "fr", url: "/post/slug-9" }, +])("seeks a menu content reference resolved for $locale", async ({ locale, url }) => { + await seedMenuReference("tg-0005"); + + const menu = await getMenuWithDb("primary", db, { locale }); + + expect(menu?.items.map((item) => item.url)).toEqual([url]); + + const queries = translationQueries(); + expect(queries.length).toBeGreaterThan(0); + for (const query of queries) { + const plan = explain(query); + expect(contentAccess(plan)).toContain("INDEX idx_ec_post_tg_locale"); + expect(plan).not.toContain("SCAN ec_post"); + expect(plan).not.toContain("idx_ec_post_loc_crt"); + expect(plan).not.toContain("idx_ec_post_loc_upd"); + } +}); + it("migrates a pre-055 table off the single-column translation_group index", async () => { sqlite.exec(`DROP INDEX idx_ec_post_tg_locale`); + sqlite.exec(`DROP INDEX idx_ec_post_del_tg_locale`); sqlite.exec(`CREATE INDEX idx_ec_post_translation_group ON ec_post (translation_group)`); captured = []; await repo.findTranslations("post", "tg-0005"); @@ -111,8 +139,12 @@ it("migrates a pre-055 table off the single-column translation_group index", asy captured = []; await repo.findTranslations("post", "tg-0005"); expect(contentAccess(explain(translationQuery()))).toMatch( - /SEARCH ec_post USING (?:COVERING )?INDEX idx_ec_post_tg_locale \(deleted_at=\? AND translation_group=\?\)/, + /SEARCH ec_post USING (?:COVERING )?INDEX idx_ec_post_del_tg_locale \(deleted_at=\? AND translation_group=\?\)/, ); + + await seedMenuReference("tg-0005"); + await getMenuWithDb("primary", db, { locale: "en" }); + expect(contentAccess(explain(translationQuery()))).toContain("INDEX idx_ec_post_tg_locale"); }); function indexNames(): string[] { @@ -123,8 +155,41 @@ function indexNames(): string[] { ).map((row) => row.name); } +async function seedMenuReference(referenceGroup: string): Promise { + for (const locale of ["en", "fr"]) { + await db + .insertInto("_emdash_menus") + .values({ id: `menu-${locale}`, name: "primary", label: "Primary", locale }) + .execute(); + await db + .insertInto("_emdash_menu_items") + .values({ + id: `item-${locale}`, + menu_id: `menu-${locale}`, + parent_id: null, + sort_order: 0, + type: "post", + reference_collection: "post", + reference_id: referenceGroup, + custom_url: null, + label: "Post", + title_attr: null, + target: null, + css_classes: null, + locale, + translation_group: null, + }) + .execute(); + } + captured = []; +} + +function translationQueries(): CapturedQuery[] { + return captured.filter((query) => query.sql.includes("translation_group")); +} + function translationQuery(): CapturedQuery { - const queries = captured.filter((query) => query.sql.includes("translation_group")); + const queries = translationQueries(); expect(queries).toHaveLength(1); return queries[0]!; } diff --git a/packages/core/tests/integration/i18n/i18n.test.ts b/packages/core/tests/integration/i18n/i18n.test.ts index df5763eaf7..bbf9723174 100644 --- a/packages/core/tests/integration/i18n/i18n.test.ts +++ b/packages/core/tests/integration/i18n/i18n.test.ts @@ -87,6 +87,7 @@ describe("i18n (Integration)", () => { const indexNames = result.rows.map((r) => r.name); expect(indexNames).toContain("idx_ec_post_locale"); expect(indexNames).toContain("idx_ec_post_tg_locale"); + expect(indexNames).toContain("idx_ec_post_del_tg_locale"); }); });