feat(vnext): authenticate CTE main query sites

Author: Light2Dark

src/vnext/__tests__/cte-layout.bench.ts

@@ -17,17 +17,19 @@ const dialect = DUCKDB_SQL_RELATION_DIALECT.cteLayout;
 
 function fixture(text: string): {
   readonly source: ReturnType<typeof createIdentitySqlSource>;
+  readonly index: ReturnType<typeof buildSqlStatementIndex>;
   readonly slot: ExactSqlStatementSlot;
 } {
   const source = createIdentitySqlSource(text);
-  const slot = buildSqlStatementIndex(
+  const index = buildSqlStatementIndex(
     source.analysisText,
     dialect.lexicalProfile,
-  ).slots[0];
+  );
+  const slot = index.slots[0];
   if (!slot || slot.boundaryQuality !== "exact") {
     throw new Error("CTE benchmark fixture requires an exact statement");
   }
-  return { slot, source };
+  return { index, slot, source };
 }
 
 const tenKibibytes = 10 * 1_024;
@@ -62,6 +64,7 @@ const bareFrameHeavyText = `SELECT ${Array.from(
 const bareFrameHeavyFixture = fixture(bareFrameHeavyText);
 const depthLayout = analyzeSqlCteLayout(
   depthHeavyFixture.source,
+  depthHeavyFixture.index,
   depthHeavyFixture.slot,
   dialect,
 );
@@ -70,6 +73,7 @@ if (depthLayout.status !== "ready") {
 }
 const projectedLayout = analyzeSqlCteLayout(
   declarationHeavyFixture.source,
+  declarationHeavyFixture.index,
   declarationHeavyFixture.slot,
   dialect,
 );
@@ -81,6 +85,7 @@ describe("CTE layout", () => {
   bench("ordinary 10 KiB statement", () => {
     analyzeSqlCteLayout(
       ordinaryFixture.source,
+      ordinaryFixture.index,
       ordinaryFixture.slot,
       dialect,
     );
@@ -89,6 +94,7 @@ describe("CTE layout", () => {
   bench("256 declarations", () => {
     analyzeSqlCteLayout(
       declarationHeavyFixture.source,
+      declarationHeavyFixture.index,
       declarationHeavyFixture.slot,
       dialect,
     );
@@ -97,6 +103,7 @@ describe("CTE layout", () => {
   bench("128-depth nested CTE", () => {
     analyzeSqlCteLayout(
       depthHeavyFixture.source,
+      depthHeavyFixture.index,
       depthHeavyFixture.slot,
       dialect,
     );
@@ -105,6 +112,7 @@ describe("CTE layout", () => {
   bench("256 sequential incomplete frames", () => {
     analyzeSqlCteLayout(
       bareFrameHeavyFixture.source,
+      bareFrameHeavyFixture.index,
       bareFrameHeavyFixture.slot,
       dialect,
     );

src/vnext/__tests__/cte-layout.test.ts

@@ -7,6 +7,7 @@ import {
   MAX_CTE_IDENTIFIER_LENGTH,
   MAX_CTE_QUOTED_IDENTIFIER_LENGTH,
   MAX_CTE_STATEMENT_LENGTH,
+  resolveAuthenticatedSqlCteEntrypoints,
   type SqlCteLayout,
   type SqlCteLayoutDialect,
   type SqlCteLayoutIssue,
@@ -55,6 +56,7 @@ function analyze(
   expect(slot?.boundaryQuality).toBe("exact");
   const result = analyzeSqlCteLayout(
     source,
+    index,
     slot as ExactSqlStatementSlot,
     dialect,
   );
@@ -68,15 +70,18 @@ function analyze(
 function analyzeRaw(
   text: string,
   dialect: SqlCteLayoutDialect = postgres,
+  indexDialect: SqlCteLayoutDialect = dialect,
 ): SqlCteLayout {
   const source = createIdentitySqlSource(text);
-  const slot = buildSqlStatementIndex(
+  const index = buildSqlStatementIndex(
     source.analysisText,
-    dialect.lexicalProfile,
-  ).slots[0];
+    indexDialect.lexicalProfile,
+  );
+  const slot = index.slots[0];
   expect(slot?.boundaryQuality).toBe("exact");
   return analyzeSqlCteLayout(
     source,
+    index,
     slot as ExactSqlStatementSlot,
     dialect,
   );
@@ -400,6 +405,7 @@ describe("bounded CTE layout", () => {
     }
     const layout = analyzeSqlCteLayout(
       source,
+      index,
       slot,
       postgres,
     );
@@ -1022,13 +1028,60 @@ describe("bounded CTE layout", () => {
         },
       },
     };
-    expect(analyzeRaw("SELECT 1", getterLexicalProfile)).toEqual({
+    expect(
+      analyzeRaw("SELECT 1", getterLexicalProfile, postgres),
+    ).toEqual({
       reason: "resource-limit",
       status: "unavailable",
     });
     expect(getterInvoked).toBe(false);
   });
 
+  it("never reads a raw dialect after validating its data descriptors", () => {
+    const text = "WITH local AS (SELECT 1) SELECT * FROM local";
+    const source = createIdentitySqlSource(text);
+    const index = buildSqlStatementIndex(
+      source.analysisText,
+      postgres.lexicalProfile,
+    );
+    const slot = index.slots[0];
+    expect(slot?.boundaryQuality).toBe("exact");
+    if (!slot || slot.boundaryQuality !== "exact") {
+      throw new Error("Expected one exact statement slot");
+    }
+
+    let rawRead = false;
+    const descriptorOnlyDialect = new Proxy(postgres, {
+      get(target, property, receiver) {
+        if (property === "lexicalProfile") {
+          rawRead = true;
+          throw new Error("hostile");
+        }
+        return Reflect.get(target, property, receiver);
+      },
+    });
+    const layout = analyzeSqlCteLayout(
+      source,
+      index,
+      slot,
+      descriptorOnlyDialect,
+    );
+
+    expect(layout.status).toBe("ready");
+    if (layout.status === "unavailable") {
+      throw new Error("Expected an authenticated CTE layout");
+    }
+    expect(
+      resolveAuthenticatedSqlCteEntrypoints(
+        layout,
+        source,
+        slot,
+        descriptorOnlyDialect,
+      ),
+    ).toEqual(layout.mainQueryEntrypoints);
+    expect(rawRead).toBe(false);
+  });
+
   it("projects only proven frame phases and validates positions", () => {
     const text =
       "WITH a AS (SELECT 1) SELECT * FROM a";

src/vnext/__tests__/incremental-statement-index.test.ts

@@ -4,6 +4,7 @@ import {
   buildSqlStatementIndex,
   DREMIO_SQL_LEXICAL_PROFILE,
   DUCKDB_SQL_LEXICAL_PROFILE,
+  isExactSqlStatementSlotSnapshotFor,
   MAX_SQL_STATEMENT_SLOTS,
   POSTGRESQL_SQL_LEXICAL_PROFILE,
   type SqlLexicalProfile,
@@ -358,18 +359,93 @@ describe("incremental statement index reuse", () => {
     },
   );
 
+  it("rebuilds when empty change metadata does not match the text", () => {
+    const previousIndex = buildSqlStatementIndex(
+      "SELECT 1",
+      POSTGRESQL_SQL_LEXICAL_PROFILE,
+    );
+    const nextIndex = updateSqlStatementIndex(
+      previousIndex,
+      "SELECT 2",
+      [],
+      POSTGRESQL_SQL_LEXICAL_PROFILE,
+    );
+    expect(nextIndex).not.toBe(previousIndex);
+    expect(nextIndex).toEqual(
+      buildSqlStatementIndex(
+        "SELECT 2",
+        POSTGRESQL_SQL_LEXICAL_PROFILE,
+      ),
+    );
+  });
+
+  it("rebuilds when the lexical profile changes", () => {
+    const text = "SELECT 1";
+    const previousIndex = buildSqlStatementIndex(
+      text,
+      DUCKDB_SQL_LEXICAL_PROFILE,
+    );
+    const nextIndex = updateSqlStatementIndex(
+      previousIndex,
+      text,
+      [],
+      POSTGRESQL_SQL_LEXICAL_PROFILE,
+    );
+    expect(nextIndex).not.toBe(previousIndex);
+    expect(nextIndex).toEqual(
+      buildSqlStatementIndex(
+        text,
+        POSTGRESQL_SQL_LEXICAL_PROFILE,
+      ),
+    );
+  });
+
+  it("rebuilds when one lexical profile object mutates", () => {
+    const profile = { ...POSTGRESQL_SQL_LEXICAL_PROFILE };
+    const text = "SELECT # ;\n SELECT 2";
+    const previousIndex = buildSqlStatementIndex(text, profile);
+    expect(previousIndex.slots).toHaveLength(2);
+    profile.hashLineComments = true;
+    const nextIndex = updateSqlStatementIndex(
+      previousIndex,
+      text,
+      [],
+      profile,
+    );
+    expect(nextIndex).not.toBe(previousIndex);
+    expect(nextIndex).toEqual(buildSqlStatementIndex(text, profile));
+    expect(nextIndex.slots).toHaveLength(1);
+  });
+
   it("reuses unchanged prefix and zero-delta suffix slot identities", () => {
     const profile = DUCKDB_SQL_LEXICAL_PROFILE;
     const text = "SELECT 1; SELECT 2; SELECT 3";
     const change = replaceFirst(text, "2", "9");
-    const { nextIndex, previousIndex } = expectIncrementalMatchesOracle(
-      text,
-      [change],
-      profile,
-    );
-    expect(expectSlot(nextIndex.slots, 0)).toBe(
-      expectSlot(previousIndex.slots, 0),
-    );
+    const { nextIndex, nextText, previousIndex } =
+      expectIncrementalMatchesOracle(
+        text,
+        [change],
+        profile,
+      );
+    const sharedPrefix = expectSlot(nextIndex.slots, 0);
+    expect(sharedPrefix).toBe(expectSlot(previousIndex.slots, 0));
+    expect(sharedPrefix.boundaryQuality).toBe("exact");
+    expect(
+      isExactSqlStatementSlotSnapshotFor(
+        previousIndex,
+        sharedPrefix,
+        text,
+        profile,
+      ),
+    ).toBe(true);
+    expect(
+      isExactSqlStatementSlotSnapshotFor(
+        nextIndex,
+        sharedPrefix,
+        nextText,
+        profile,
+      ),
+    ).toBe(true);
     expect(expectSlot(nextIndex.slots, 1)).not.toBe(
       expectSlot(previousIndex.slots, 1),
     );