feat(vnext): unify built-in relation dialect policy (#189)

## Summary

- add one package-owned relation dialect runtime for PostgreSQL, DuckDB,
BigQuery, and Dremio
- unify identifier decoding, CTE equality/prefix rules, query-path
decoding, role-aware rendering, reserved words, and CTE/query recognizer
policy
- authenticate each runtime through the opaque built-in dialect handle
while keeping callbacks private
- add hostile-input bounds and fail-closed behavior, exact dialect
corpora, production-backed recognizer tests, and bundle budgets

## Correctness and performance

- changed production coverage: 97.10% statements, 96.18% branches, 100%
functions, 97.05% lines
- exact packed core: 15,028 gzip / 47,141 raw bytes under 16 KiB / 52
KiB limits
- exact packed worker output: 125,937 gzip / 572,982 raw bytes under 128
KiB / 570 KiB limits
- comparator hot paths use bounded allocation-free ASCII loops; private
path scratch arrays are not copied or frozen

## Verification

- 1,646 tests passed, 1 governed expected failure
- all five TypeScript configurations passed
- oxlint and test-integrity checks passed
- browser suite passed in Chromium
- exact tarball package smoke and SSR import passed
- worker placement, strict CSP, lazy loading, and parser isolation
passed
- query-site and CTE benchmarks passed
- exact head 6e80f2f approved
independently by three adversarial reviewers

Part of #169.

<!-- This is an auto-generated description by cubic. -->
---
## Summary by cubic
Unifies the relation dialect runtime across PostgreSQL, DuckDB,
BigQuery, and Dremio with a single, package-owned policy for
identifiers, CTEs, query paths, and role-aware rendering. Tightens the
public API and enforces strict size budgets and hostile-input
boundaries. Part of #169.

- **New Features**
- Added a unified `relation-dialect` runtime for completion, CTE layout,
query-site policy, reserved words, and rendering across all built-ins.
- Introduced fail-closed decoding with hostile-input bounds and exact
corpora; added adversarial/production-backed tests.
- Enforced bundle budgets (core ≤16 KiB gzip/52 KiB raw; worker ≤128 KiB
gzip/570 KiB raw) and a smoke check to prevent dialect policy leaks.

- **Refactors**
- Public dialect handles (`bigQueryDialect()`, `dremioDialect()`,
`duckdbDialect()`, `postgresDialect()`) now expose only `displayName`
and `id`; callbacks are private and authenticated in `session`.
- Migrated `cte-layout` and `query-site` to `..._SQL_RELATION_DIALECT`
constants; added `MAX_CTE_QUOTED_IDENTIFIER_LENGTH`.
- Simplified and hardened tests by removing ad-hoc dialects; updated
worker placement and docs to report core/worker sizes.

<sup>Written for commit 6e80f2f.
Summary will update on new commits.</sup>

<a
href="https://cubic.dev/pr/marimo-team/codemirror-sql/pull/189?utm_source=github"
target="_blank" rel="noopener noreferrer"
data-no-image-dialog="true"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img
alt="Review in cubic"
src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>

<!-- End of auto-generated description by cubic. -->

Author: Light2Dark

scripts/package-smoke.mjs

@@ -296,6 +296,22 @@ if (
 ) {
   throw new Error("vNext package exports are incomplete");
 }
+for (const dialect of [
+  vnext.bigQueryDialect(),
+  vnext.dremioDialect(),
+  vnext.duckdbDialect(),
+  vnext.postgresDialect(),
+]) {
+  if (
+    Object.keys(dialect).join(",") !== "displayName,id" ||
+    "decodeIdentifier" in dialect ||
+    "grammar" in dialect ||
+    "relationDialect" in dialect ||
+    "renderRelationPath" in dialect
+  ) {
+    throw new Error("vNext dialect implementation policy leaked publicly");
+  }
+}
 if (!commonKeywords.keywords || !duckdbKeywords.keywords) {
   throw new Error("Keyword data exports are incomplete");
 }

scripts/worker-placement.mjs

@@ -35,8 +35,10 @@ const PARSER_MARKERS = [
   "tableList",
 ];
 const BIGQUERY_GZIP_LIMIT = 50 * 1024;
+const CORE_TOTAL_GZIP_LIMIT = 16 * 1024;
+const CORE_TOTAL_RAW_LIMIT = 52 * 1024;
 const POSTGRESQL_GZIP_LIMIT = 68 * 1024;
-const WORKER_TOTAL_GZIP_LIMIT = 120 * 1024;
+const WORKER_TOTAL_GZIP_LIMIT = 128 * 1024;
 const WORKER_TOTAL_RAW_LIMIT = 570 * 1024;
 const MIME_TYPES = new Map([
   [".css", "text/css; charset=utf-8"],
@@ -485,6 +487,27 @@ function verifyCoreExcludesParser(coreDirectory) {
   return new Set(moduleIds).size;
 }
 
+function verifyCoreSize(coreDirectory) {
+  const report = bundleReport(coreDirectory);
+  if (report.gzipBytes > CORE_TOTAL_GZIP_LIMIT) {
+    throw new Error(
+      `Core output exceeded ${CORE_TOTAL_GZIP_LIMIT} gzip bytes: ${report.gzipBytes}`,
+    );
+  }
+  if (report.rawBytes > CORE_TOTAL_RAW_LIMIT) {
+    throw new Error(
+      `Core output exceeded ${CORE_TOTAL_RAW_LIMIT} raw bytes: ${report.rawBytes}`,
+    );
+  }
+  return {
+    ...report,
+    limits: {
+      gzipBytes: CORE_TOTAL_GZIP_LIMIT,
+      rawBytes: CORE_TOTAL_RAW_LIMIT,
+    },
+  };
+}
+
 function verifySsrImport(fixtureDirectory) {
   const source = `
 Object.defineProperty(globalThis, "window", {
@@ -751,6 +774,7 @@ try {
   const coreDirectory = join(fixtureDirectory, "core-dist");
   const workersDirectory = join(fixtureDirectory, "workers-dist");
   const coreModuleCount = verifyCoreExcludesParser(coreDirectory);
+  const coreBundle = verifyCoreSize(coreDirectory);
   const chromiumResult = await runChromium(
     fixtureDirectory,
     workersDirectory,
@@ -763,7 +787,7 @@ try {
   }
   const report = {
     bundles: {
-      core: bundleReport(coreDirectory),
+      core: coreBundle,
       workers: workerBundles,
     },
     evidence: {

src/vnext/__tests__/cte-layout.bench.ts

@@ -4,35 +4,16 @@ import {
   MAX_CTE_DECLARATIONS,
   MAX_CTE_DEPTH,
   MAX_CTE_FRAMES,
-  type SqlCteLayoutDialect,
   visibleSqlCtesAt,
 } from "../cte-layout.js";
-import { DUCKDB_SQL_LEXICAL_PROFILE } from "../lexical.js";
+import { DUCKDB_SQL_RELATION_DIALECT } from "../relation-dialect.js";
 import { createIdentitySqlSource } from "../source.js";
 import {
   buildSqlStatementIndex,
   type ExactSqlStatementSlot,
 } from "../statement-index.js";
 
-const dialect: SqlCteLayoutDialect = {
-  classifyIdentifierToken: (rawIdentifier, quoted) => ({
-    status: "identifier",
-    value: {
-      component: { quoted, value: rawIdentifier },
-    },
-  }),
-  compareCteIdentifiers: (left, right) =>
-    left.value.toLowerCase() === right.value.toLowerCase()
-      ? "equal"
-      : "distinct",
-  grammar: {
-    declaredColumns: true,
-    materialization: true,
-    maximumDeclarationsPerFrame: MAX_CTE_DECLARATIONS,
-    recursive: true,
-  },
-  lexicalProfile: DUCKDB_SQL_LEXICAL_PROFILE,
-};
+const dialect = DUCKDB_SQL_RELATION_DIALECT.cteLayout;
 
 function fixture(text: string): {
   readonly source: ReturnType<typeof createIdentitySqlSource>;

src/vnext/__tests__/cte-layout.test.ts

@@ -5,6 +5,7 @@ import {
   MAX_CTE_DEPTH,
   MAX_CTE_FRAMES,
   MAX_CTE_IDENTIFIER_LENGTH,
+  MAX_CTE_QUOTED_IDENTIFIER_LENGTH,
   MAX_CTE_STATEMENT_LENGTH,
   type SqlCteLayout,
   type SqlCteLayoutDialect,
@@ -13,11 +14,11 @@ import {
   visibleSqlCtesAt,
 } from "../cte-layout.js";
 import {
-  BIGQUERY_SQL_LEXICAL_PROFILE,
-  DREMIO_SQL_LEXICAL_PROFILE,
-  DUCKDB_SQL_LEXICAL_PROFILE,
-  POSTGRESQL_SQL_LEXICAL_PROFILE,
-} from "../lexical.js";
+  BIGQUERY_SQL_RELATION_DIALECT,
+  DREMIO_SQL_RELATION_DIALECT,
+  DUCKDB_SQL_RELATION_DIALECT,
+  POSTGRESQL_SQL_RELATION_DIALECT,
+} from "../relation-dialect.js";
 import {
   createIdentitySqlSource,
   createMaskedSqlSource,
@@ -28,94 +29,10 @@ import {
   type ExactSqlStatementSlot,
 } from "../statement-index.js";
 
-function decodeQuoted(raw: string): string {
-  const quote = raw.at(0) ?? "";
-  return raw
-    .slice(1, -1)
-    .replaceAll(`${quote}${quote}`, quote)
-    .replaceAll(`\\${quote}`, quote);
-}
-
-function isAscii(value: string): boolean {
-  for (let index = 0; index < value.length; index += 1) {
-    if (value.charCodeAt(index) > 0x7f) {
-      return false;
-    }
-  }
-  return true;
-}
-
-function createDialect(
-  kind: "bigquery" | "dremio" | "duckdb" | "postgresql",
-): SqlCteLayoutDialect {
-  const lexicalProfile =
-    kind === "bigquery"
-      ? BIGQUERY_SQL_LEXICAL_PROFILE
-      : kind === "dremio"
-        ? DREMIO_SQL_LEXICAL_PROFILE
-        : kind === "duckdb"
-          ? DUCKDB_SQL_LEXICAL_PROFILE
-          : POSTGRESQL_SQL_LEXICAL_PROFILE;
-  return {
-    classifyIdentifierToken: (raw, quoted) => {
-      const value = quoted ? decodeQuoted(raw) : raw;
-      if (
-        value.length === 0 ||
-        (!quoted &&
-          /^(?:as|materialized|not|recursive|select|with)$/i.test(
-            value,
-          ))
-      ) {
-        return { status: "unsupported" };
-      }
-      return {
-        status: "identifier",
-        value: {
-          component: { quoted, value },
-        },
-      };
-    },
-    compareCteIdentifiers: (left, right) => {
-      const leftKey =
-        kind === "postgresql" && left.quoted
-          ? isAscii(left.value)
-            ? left.value
-            : null
-          : isAscii(left.value)
-            ? left.value.toLowerCase()
-            : null;
-      const rightKey =
-        kind === "postgresql" && right.quoted
-          ? isAscii(right.value)
-            ? right.value
-            : null
-          : isAscii(right.value)
-            ? right.value.toLowerCase()
-            : null;
-      if (leftKey !== null && rightKey !== null) {
-        return leftKey === rightKey ? "equal" : "distinct";
-      }
-      return left.quoted === right.quoted &&
-        left.value === right.value
-        ? "equal"
-        : "unknown";
-    },
-    grammar: {
-      declaredColumns: kind !== "bigquery",
-      materialization:
-        kind === "duckdb" || kind === "postgresql",
-      maximumDeclarationsPerFrame:
-        kind === "dremio" ? 1 : MAX_CTE_DECLARATIONS,
-      recursive: kind !== "dremio",
-    },
-    lexicalProfile,
-  };
-}
-
-const postgres = createDialect("postgresql");
-const duckdb = createDialect("duckdb");
-const bigquery = createDialect("bigquery");
-const dremio = createDialect("dremio");
+const postgres = POSTGRESQL_SQL_RELATION_DIALECT.cteLayout;
+const duckdb = DUCKDB_SQL_RELATION_DIALECT.cteLayout;
+const bigquery = BIGQUERY_SQL_RELATION_DIALECT.cteLayout;
+const dremio = DREMIO_SQL_RELATION_DIALECT.cteLayout;
 
 function analyze(
   text: string,
@@ -787,7 +704,7 @@ describe("bounded CTE layout", () => {
     expect(calls).toBe(0);
 
     const oversizedQuoted = `"${"a".repeat(
-      MAX_CTE_IDENTIFIER_LENGTH * 2 + 1,
+      MAX_CTE_QUOTED_IDENTIFIER_LENGTH,
     )}"`;
     expectPartial(
       `WITH ${oversizedQuoted} AS (SELECT 1) SELECT 1`,

src/vnext/__tests__/query-site.bench.ts

@@ -1,34 +1,15 @@
 import { bench, describe } from "vitest";
 import {
   recognizeSqlRelationQuerySite,
-  type SqlQuerySiteDialect,
 } from "../query-site.js";
+import { DUCKDB_SQL_RELATION_DIALECT } from "../relation-dialect.js";
 import { createIdentitySqlSource } from "../source.js";
 import {
   buildSqlStatementIndex,
-  DUCKDB_SQL_LEXICAL_PROFILE,
   findSqlStatementSlot,
 } from "../statement-index.js";
 
-const dialect: SqlQuerySiteDialect = {
-  classifyIdentifierToken: (rawIdentifier) => ({
-    status: "identifier",
-    value: rawIdentifier,
-  }),
-  decodeRelationPath: (rawPath, cursorOffset) => ({
-    finalSegment: { from: 0, to: rawPath.length },
-    prefix: {
-      quoted: false,
-      value: rawPath.slice(0, cursorOffset),
-    },
-    qualifier: [],
-    quality: "exact",
-    status: "decoded",
-  }),
-  lexicalProfile: DUCKDB_SQL_LEXICAL_PROFILE,
-  maximumPathDepth: 16,
-  supportsNaturalJoin: true,
-};
+const dialect = DUCKDB_SQL_RELATION_DIALECT.querySite;
 const TEN_KIBIBYTES = 10 * 1_024;
 const queryPrefix = "SELECT ";
 const querySuffix = " FROM schema_prefix";