fix(core): split SEO fetch from loadEntry to avoid D1 result-set column limit on wide collections

.changeset/fix-d1-result-set-column-limit.md

@@ -0,0 +1,5 @@
+---
+"emdash": patch
+---
+
+Fixes silent `null` entries on wide-schema collections under Cloudflare D1. The content loader's single-query `LEFT JOIN _emdash_seo` added 5 alias columns to every result set, which pushed collections with ~95+ flat user fields past D1's per-query column limit (~100). The query failed with `D1_ERROR: too many columns in result set`, the error was wrapped as a generic `Failed to load entry`, and the call site surfaced `null`. SEO is now folded into a single aggregated JSON column (mirroring how byline and taxonomy hydration already work), keeping the result-set width bounded regardless of how wide the collection schema gets while preserving the single round trip.

packages/core/src/loader.ts

@@ -25,15 +25,23 @@ import { isMissingColumnError, isMissingTableError } from "./utils/db-errors.js"
 const FIELD_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
 
 /**
- * SEO columns joined in from `_emdash_seo` on the single-entry path, mapped to
- * aliased result keys. SEO lives in a side table, so a LEFT JOIN folds it into
- * the entry load at zero extra query cost; the result is surfaced as a nested
- * `data.seo` object (see extractSeo) rather than flat fields.
+ * SEO columns folded into the single-entry query as a single JSON column
+ * (`_emdash_seo` in the result set), then expanded onto the row under these
+ * aliases for `extractSeo()`. Surfacing SEO as one aggregated column keeps the
+ * result-set width bounded regardless of how many fields the collection has,
+ * which matters for D1: a flat `LEFT JOIN _emdash_seo` adds 5 alias columns to
+ * every row and pushes wide collections (common after WordPress / ACF imports)
+ * past D1's per-result-set column limit, surfacing as a silent null entry.
+ * One JSON column is one column, so the join stays safe at any schema width.
+ *
+ * The aliases mirror the strategy used by `foldedHydrationSelects` for byline
+ * and taxonomy hydration: aggregate in SQL, expand in JS. SEO is 1:1 with
+ * content, so the subquery uses `json_object` (not the array aggregator).
  *
  * The `_emdash_` prefix on the aliases guarantees they can never collide with
  * a content field. Field slugs must match `/^[a-z][a-z0-9_]*$/`, so a user can
- * legitimately define a `seo_title` field; selecting the joined column under
- * its bare name would shadow that field in the result set and drop the user's
+ * legitimately define a `seo_title` field; surfacing the SEO column under its
+ * bare name would shadow that field in the result set and drop the user's
  * value. The prefix (illegal as a leading slug char) sidesteps this entirely.
  */
 const SEO_COLUMN_ALIASES: Record<string, string> = {
@@ -47,6 +55,9 @@ const SEO_COLUMN_ALIASES: Record<string, string> = {
 /** Aliased SEO result keys — excluded from generic field mapping. */
 const SEO_ALIAS_COLUMNS = Object.values(SEO_COLUMN_ALIASES);
 
+/** Folded SEO JSON column name in the result set (expanded onto aliases in JS). */
+const SEO_FOLDED_COLUMN = "_emdash_seo";
+
 /**
  * System columns excluded from entry.data
  * Note: slug is intentionally NOT excluded - it's useful as data.slug in templates
@@ -67,15 +78,17 @@ const SYSTEM_COLUMNS = new Set([
 	"draft_revision_id",
 	"locale",
 	"translation_group",
-	// Aliased SEO columns joined from _emdash_seo on the single-entry path.
-	// Surfaced as a nested data.seo object (see extractSeo), never as flat
-	// fields. The aliases are _emdash_-prefixed so they can't shadow a user
-	// field named e.g. `seo_title`.
+	// Aliased SEO columns expanded from the folded _emdash_seo JSON column on
+	// the single-entry path. Surfaced as a nested data.seo object (see
+	// extractSeo), never as flat fields. The aliases are _emdash_-prefixed so
+	// they can't shadow a user field named e.g. `seo_title`.
 	...SEO_ALIAS_COLUMNS,
-	// Folded hydration JSON columns (see foldedHydrationSelects) — surfaced via
-	// the FOLDED_* markers, never as flat fields.
+	// Folded hydration JSON columns (see foldedHydrationSelects and
+	// foldedSeoSelect) — surfaced via the FOLDED_* markers or expanded onto
+	// SEO_ALIAS_COLUMNS, never as flat fields.
 	"_emdash_terms",
 	"_emdash_bylines",
+	SEO_FOLDED_COLUMN,
 ]);
 
 /** Markers for byline/taxonomy hydration folded into the content query. */
@@ -123,6 +136,62 @@ function foldedHydrationSelects(db: Kysely<any>, type: string, outer: string) {
 	return { terms, bylines };
 }
 
+/**
+ * Correlated JSON-object subquery that folds per-entry SEO into the content
+ * query without widening the result set: 1 row of `_emdash_seo` becomes 1 JSON
+ * column rather than 5 flat columns. The JSON column is expanded onto the row
+ * via {@link expandFoldedSeo} after the query runs, preserving the alias keys
+ * that {@link extractSeo} reads. Missing SEO row (no entry in `_emdash_seo`)
+ * yields NULL, which {@link expandFoldedSeo} treats as "no SEO" - identical to
+ * the prior LEFT JOIN miss behavior.
+ *
+ * Dialect-specific aggregation mirrors {@link foldedHydrationSelects}: SQLite
+ * `json_object` returns a JSON *string*, Postgres `json_build_object` returns
+ * parsed JSON; both branches are handled in expansion.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- any Kysely instance
+function foldedSeoSelect(db: Kysely<any>, type: string, outer: string) {
+	const o = sql.ref(outer);
+	const pg = isPostgres(db);
+	// Use raw column names (not aliases) as JSON keys: the JSON is expanded back
+	// onto SEO_COLUMN_ALIASES in JS, and keeping the keys matched to the
+	// underlying columns makes the SQL readable and the expansion 1-to-1.
+	const pairs =
+		"'seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index";
+	const obj = pg ? sql.raw(`json_build_object(${pairs})`) : sql.raw(`json_object(${pairs})`);
+	return sql`(SELECT ${obj} FROM ${sql.ref("_emdash_seo")} AS s WHERE s.collection = ${type} AND s.content_id = ${o}.id LIMIT 1) AS ${sql.ref(SEO_FOLDED_COLUMN)}`;
+}
+
+/**
+ * Expand the folded `_emdash_seo` JSON column onto the row using SEO_COLUMN_ALIASES,
+ * so {@link extractSeo} reads it transparently. SQLite returns a JSON string
+ * (parse it); Postgres returns already-parsed JSON. Missing/malformed/null is
+ * a no-op: {@link extractSeo} returns null when the aliases are absent.
+ */
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+	return !!value && typeof value === "object" && !Array.isArray(value);
+}
+
+function expandFoldedSeo(row: Record<string, unknown>): void {
+	const raw = row[SEO_FOLDED_COLUMN];
+	delete row[SEO_FOLDED_COLUMN];
+	let parsed: Record<string, unknown> | null = null;
+	if (typeof raw === "string") {
+		try {
+			const candidate: unknown = JSON.parse(raw);
+			if (isPlainObject(candidate)) parsed = candidate;
+		} catch {
+			return; // malformed JSON: leave the row without SEO aliases (extractSeo returns null)
+		}
+	} else if (isPlainObject(raw)) {
+		parsed = raw;
+	}
+	if (!parsed) return;
+	for (const [col, alias] of Object.entries(SEO_COLUMN_ALIASES)) {
+		row[alias] = parsed[col] ?? null;
+	}
+}
+
 /**
  * Stash folded hydration JSON (non-enumerable) for the query.ts fast paths.
  * SQLite returns a JSON string (parse it); Postgres returns already-parsed JSON.
@@ -1050,40 +1119,32 @@ export function emdashLoader(): LiveLoader<EntryData, EntryFilter, CollectionFil
 				// When locale is specified, prefer locale-scoped slug match,
 				// but IDs are globally unique so always check id without locale scope.
 				//
-				// LEFT JOIN _emdash_seo folds per-entry SEO (canonical, noindex,
-				// etc.) into this single query at zero extra round-trip cost. The
-				// joined columns are surfaced as a nested data.seo object via
-				// extractSeo() and excluded from the generic field mapping. SEO is
-				// 1:1 with content (PK on collection+content_id), so the join never
-				// multiplies rows.
-				const seoSelect = sql.join(
-					Object.entries(SEO_COLUMN_ALIASES).map(
-						([col, alias]) => sql`${sql.ref(`s.${col}`)} AS ${sql.ref(alias)}`,
-					),
-				);
-				// Fold byline + taxonomy hydration into the content query (see
-				// foldedHydrationSelects), removing the two separate hydration
-				// round trips per fetch.
+				// Byline + taxonomy hydration (foldedHydrationSelects) and per-entry
+				// SEO (foldedSeoSelect) are each surfaced as a single aggregated JSON
+				// column rather than flat columns. This keeps the result-set width
+				// bounded at any collection schema width: a flat `LEFT JOIN _emdash_seo`
+				// adds 5 alias columns to every row and pushes wide flat-schema
+				// collections (common after WordPress / ACF imports) past D1's
+				// per-result-set column limit, surfacing as a silent null entry. One
+				// JSON column is one column, so the join stays safe at any width and
+				// we keep the single round trip.
 				const { terms: termsSelect, bylines: bylinesSelect } = foldedHydrationSelects(
 					db,
 					type,
 					"c",
 				);
+				const seoSelect = foldedSeoSelect(db, type, "c");
 				const result = locale
 					? await sql<Record<string, unknown>>`
 							SELECT c.*, ${seoSelect}, ${termsSelect}, ${bylinesSelect}
 							FROM ${sql.ref(tableName)} AS c
-							LEFT JOIN ${sql.ref("_emdash_seo")} AS s
-								ON s.collection = ${type} AND s.content_id = c.id
 							WHERE c.deleted_at IS NULL
 							AND ((c.slug = ${id} AND c.locale = ${locale}) OR c.id = ${id})
 							LIMIT 1
 						`.execute(db)
 					: await sql<Record<string, unknown>>`
 							SELECT c.*, ${seoSelect}, ${termsSelect}, ${bylinesSelect}
 							FROM ${sql.ref(tableName)} AS c
-							LEFT JOIN ${sql.ref("_emdash_seo")} AS s
-								ON s.collection = ${type} AND s.content_id = c.id
 							WHERE c.deleted_at IS NULL
 							AND (c.slug = ${id} OR c.id = ${id})
 							LIMIT 1
@@ -1094,6 +1155,11 @@ export function emdashLoader(): LiveLoader<EntryData, EntryFilter, CollectionFil
 					return undefined;
 				}
 
+				// Expand the folded SEO JSON column onto SEO_COLUMN_ALIASES so
+				// extractSeo() reads it transparently. Missing/null SEO is a
+				// no-op: extractSeo() returns null when the aliases are absent.
+				expandFoldedSeo(row);
+
 				const i18nConfig = virtualConfig?.i18n;
 				const i18nEnabled = i18nConfig && i18nConfig.locales.length > 1;
 				const entrySlug = rowStr(row, "slug") || rowStr(row, "id");

packages/core/tests/unit/loader-seo.test.ts

@@ -14,10 +14,12 @@ import {
 /**
  * Regression test for #1270: SEO fields (noindex toggle, canonical URL) set in
  * the admin had no effect on rendered pages because the content loader never
- * surfaced the `_emdash_seo` row. The loader now LEFT JOINs that table and
- * attaches the result to `entry.data.seo`, which `getSeoMeta()` reads.
+ * surfaced the `_emdash_seo` row. The loader now folds that row into the
+ * single-entry query as an aggregated JSON column and attaches the expanded
+ * result to `entry.data.seo`, which `getSeoMeta()` reads.
  *
- * Run on both dialects — the LEFT JOIN SQL is dialect-sensitive.
+ * Run on both dialects, since the JSON aggregation SQL is dialect-sensitive
+ * (`json_object` on SQLite, `json_build_object` on Postgres).
  */
 describeEachDialect("Loader SEO hydration (#1270)", (dialect) => {
 	let ctx: DialectTestContext;

packages/core/tests/unit/loader-wide-collection.test.ts

@@ -0,0 +1,157 @@
+import { it, expect, beforeEach, afterEach } from "vitest";
+
+import { handleContentCreate } from "../../src/api/index.js";
+import { SeoRepository } from "../../src/database/repositories/seo.js";
+import { emdashLoader } from "../../src/loader.js";
+import { runWithContext } from "../../src/request-context.js";
+import { SchemaRegistry } from "../../src/schema/registry.js";
+import {
+	describeEachDialect,
+	setupForDialect,
+	teardownForDialect,
+	type DialectTestContext,
+} from "../utils/test-db.js";
+
+/**
+ * Regression test for #1600: loadEntry's SELECT shape on wide collections.
+ *
+ * When a per-collection `ec_*` table has many flat scalar columns (common when
+ * porting from WordPress / ACF or other builders where every section is a
+ * top-level field), the previous implementation did:
+ *
+ *   SELECT c.*, <5 SEO alias columns> FROM ec_table c LEFT JOIN _emdash_seo s
+ *
+ * On Cloudflare D1 the per-query result-set column limit (~100) made this
+ * fail with `D1_ERROR: too many columns in result set` for collections
+ * around 95+ user columns. The loader's try/catch wrapped it as a generic
+ * `Failed to load entry` error and the call site returned a silent `null`.
+ *
+ * The fix folds SEO into a single aggregated JSON column (`_emdash_seo`)
+ * mirroring how byline and taxonomy hydration already work: aggregate in SQL,
+ * expand in JS. One JSON column is one column, so the result-set width stays
+ * bounded at any collection schema width and the single round trip is
+ * preserved.
+ *
+ * Run on both dialects to keep parity with loader-seo.test.ts.
+ */
+describeEachDialect("Loader on wide-schema collections (#1600)", (dialect) => {
+	let ctx: DialectTestContext;
+	let seoRepo: SeoRepository;
+	const COLLECTION = "wide_collection";
+	const USER_FIELD_COUNT = 95;
+
+	beforeEach(async () => {
+		ctx = await setupForDialect(dialect);
+		const registry = new SchemaRegistry(ctx.db);
+
+		// Create a collection with SEO enabled and a large number of flat
+		// scalar fields. 95 user fields + 14 system columns + 5 SEO aliases
+		// would have been ~114 result-set columns under the old LEFT JOIN
+		// shape, well past D1's per-query limit.
+		await registry.createCollection({
+			slug: COLLECTION,
+			label: "Wide Collection",
+			labelSingular: "Wide Entry",
+		});
+		await registry.createField(COLLECTION, {
+			slug: "title",
+			label: "Title",
+			type: "string",
+		});
+		for (let i = 1; i <= USER_FIELD_COUNT; i++) {
+			await registry.createField(COLLECTION, {
+				slug: `field_${i}`,
+				label: `Field ${i}`,
+				type: "string",
+			});
+		}
+		// Enable SEO so extractSeo() has somewhere to read from.
+		await ctx.db
+			.updateTable("_emdash_collections")
+			.set({ has_seo: 1 })
+			.where("slug", "=", COLLECTION)
+			.execute();
+
+		seoRepo = new SeoRepository(ctx.db);
+	});
+
+	afterEach(async () => {
+		await teardownForDialect(ctx);
+	});
+
+	function load(idOrSlug: string) {
+		const loader = emdashLoader();
+		return runWithContext({ db: ctx.db }, () =>
+			loader.loadEntry!({ filter: { type: COLLECTION, id: idOrSlug } }),
+		);
+	}
+
+	it("loads an entry from a collection with 95+ flat user columns", async () => {
+		const data: Record<string, string> = { title: "Wide Entry" };
+		for (let i = 1; i <= USER_FIELD_COUNT; i++) {
+			data[`field_${i}`] = `value-${i}`;
+		}
+		const result = await handleContentCreate(ctx.db, COLLECTION, {
+			data,
+			status: "published",
+		});
+		if (!result.success) throw new Error("Failed to create entry");
+		const slug = result.data!.item.slug!;
+
+		const loaded = await load(slug);
+
+		expect(loaded).toBeDefined();
+		expect((loaded as { data: Record<string, unknown> }).data.title).toBe("Wide Entry");
+		// Spot-check a handful of user fields across the range.
+		const loadedData = (loaded as { data: Record<string, unknown> }).data;
+		expect(loadedData.field_1).toBe("value-1");
+		expect(loadedData.field_50).toBe("value-50");
+		expect(loadedData.field_95).toBe("value-95");
+	});
+
+	it("still attaches data.seo on wide collections (folded JSON column)", async () => {
+		const data: Record<string, string> = { title: "Wide With SEO" };
+		for (let i = 1; i <= USER_FIELD_COUNT; i++) {
+			data[`field_${i}`] = `value-${i}`;
+		}
+		const result = await handleContentCreate(ctx.db, COLLECTION, {
+			data,
+			status: "published",
+		});
+		if (!result.success) throw new Error("Failed to create entry");
+		const item = result.data!.item;
+
+		await seoRepo.upsert(COLLECTION, item.id, {
+			noIndex: true,
+			canonical: "https://example.com/wide",
+			title: "Wide SEO Title",
+		});
+
+		const loaded = await load(item.slug!);
+		const loadedData = (loaded as { data: Record<string, unknown> }).data;
+		const seo = loadedData.seo as Record<string, unknown> | undefined;
+
+		expect(seo).toBeDefined();
+		expect(seo!.noIndex).toBe(true);
+		expect(seo!.canonical).toBe("https://example.com/wide");
+		expect(seo!.title).toBe("Wide SEO Title");
+	});
+
+	it("omits data.seo when no SEO row exists, even on wide collections", async () => {
+		const data: Record<string, string> = { title: "No SEO" };
+		for (let i = 1; i <= USER_FIELD_COUNT; i++) {
+			data[`field_${i}`] = `value-${i}`;
+		}
+		const result = await handleContentCreate(ctx.db, COLLECTION, {
+			data,
+			status: "published",
+		});
+		if (!result.success) throw new Error("Failed to create entry");
+		const slug = result.data!.item.slug!;
+
+		const loaded = await load(slug);
+		const loadedData = (loaded as { data: Record<string, unknown> }).data;
+
+		expect(loadedData.seo).toBeUndefined();
+	});
+});