diff --git a/src/vnext/__tests__/cte-layout.bench.ts b/src/vnext/__tests__/cte-layout.bench.ts index 0d179b5..945223b 100644 --- a/src/vnext/__tests__/cte-layout.bench.ts +++ b/src/vnext/__tests__/cte-layout.bench.ts @@ -17,17 +17,19 @@ const dialect = DUCKDB_SQL_RELATION_DIALECT.cteLayout; function fixture(text: string): { readonly source: ReturnType; + readonly index: ReturnType; readonly slot: ExactSqlStatementSlot; } { const source = createIdentitySqlSource(text); - const slot = buildSqlStatementIndex( + const index = buildSqlStatementIndex( source.analysisText, dialect.lexicalProfile, - ).slots[0]; + ); + const slot = index.slots[0]; if (!slot || slot.boundaryQuality !== "exact") { throw new Error("CTE benchmark fixture requires an exact statement"); } - return { slot, source }; + return { index, slot, source }; } const tenKibibytes = 10 * 1_024; @@ -62,6 +64,7 @@ const bareFrameHeavyText = `SELECT ${Array.from( const bareFrameHeavyFixture = fixture(bareFrameHeavyText); const depthLayout = analyzeSqlCteLayout( depthHeavyFixture.source, + depthHeavyFixture.index, depthHeavyFixture.slot, dialect, ); @@ -70,6 +73,7 @@ if (depthLayout.status !== "ready") { } const projectedLayout = analyzeSqlCteLayout( declarationHeavyFixture.source, + declarationHeavyFixture.index, declarationHeavyFixture.slot, dialect, ); @@ -81,6 +85,7 @@ describe("CTE layout", () => { bench("ordinary 10 KiB statement", () => { analyzeSqlCteLayout( ordinaryFixture.source, + ordinaryFixture.index, ordinaryFixture.slot, dialect, ); @@ -89,6 +94,7 @@ describe("CTE layout", () => { bench("256 declarations", () => { analyzeSqlCteLayout( declarationHeavyFixture.source, + declarationHeavyFixture.index, declarationHeavyFixture.slot, dialect, ); @@ -97,6 +103,7 @@ describe("CTE layout", () => { bench("128-depth nested CTE", () => { analyzeSqlCteLayout( depthHeavyFixture.source, + depthHeavyFixture.index, depthHeavyFixture.slot, dialect, ); @@ -105,6 +112,7 @@ describe("CTE layout", () => { bench("256 sequential incomplete frames", () => { analyzeSqlCteLayout( bareFrameHeavyFixture.source, + bareFrameHeavyFixture.index, bareFrameHeavyFixture.slot, dialect, ); diff --git a/src/vnext/__tests__/cte-layout.test.ts b/src/vnext/__tests__/cte-layout.test.ts index 97ce704..213e861 100644 --- a/src/vnext/__tests__/cte-layout.test.ts +++ b/src/vnext/__tests__/cte-layout.test.ts @@ -7,6 +7,7 @@ import { MAX_CTE_IDENTIFIER_LENGTH, MAX_CTE_QUOTED_IDENTIFIER_LENGTH, MAX_CTE_STATEMENT_LENGTH, + resolveAuthenticatedSqlCteEntrypoints, type SqlCteLayout, type SqlCteLayoutDialect, type SqlCteLayoutIssue, @@ -55,6 +56,7 @@ function analyze( expect(slot?.boundaryQuality).toBe("exact"); const result = analyzeSqlCteLayout( source, + index, slot as ExactSqlStatementSlot, dialect, ); @@ -68,15 +70,18 @@ function analyze( function analyzeRaw( text: string, dialect: SqlCteLayoutDialect = postgres, + indexDialect: SqlCteLayoutDialect = dialect, ): SqlCteLayout { const source = createIdentitySqlSource(text); - const slot = buildSqlStatementIndex( + const index = buildSqlStatementIndex( source.analysisText, - dialect.lexicalProfile, - ).slots[0]; + indexDialect.lexicalProfile, + ); + const slot = index.slots[0]; expect(slot?.boundaryQuality).toBe("exact"); return analyzeSqlCteLayout( source, + index, slot as ExactSqlStatementSlot, dialect, ); @@ -400,6 +405,7 @@ describe("bounded CTE layout", () => { } const layout = analyzeSqlCteLayout( source, + index, slot, postgres, ); @@ -1022,13 +1028,60 @@ describe("bounded CTE layout", () => { }, }, }; - expect(analyzeRaw("SELECT 1", getterLexicalProfile)).toEqual({ + expect( + analyzeRaw("SELECT 1", getterLexicalProfile, postgres), + ).toEqual({ reason: "resource-limit", status: "unavailable", }); expect(getterInvoked).toBe(false); }); + it("never reads a raw dialect after validating its data descriptors", () => { + const text = "WITH local AS (SELECT 1) SELECT * FROM local"; + const source = createIdentitySqlSource(text); + const index = buildSqlStatementIndex( + source.analysisText, + postgres.lexicalProfile, + ); + const slot = index.slots[0]; + expect(slot?.boundaryQuality).toBe("exact"); + if (!slot || slot.boundaryQuality !== "exact") { + throw new Error("Expected one exact statement slot"); + } + + let rawRead = false; + const descriptorOnlyDialect = new Proxy(postgres, { + get(target, property, receiver) { + if (property === "lexicalProfile") { + rawRead = true; + throw new Error("hostile"); + } + return Reflect.get(target, property, receiver); + }, + }); + const layout = analyzeSqlCteLayout( + source, + index, + slot, + descriptorOnlyDialect, + ); + + expect(layout.status).toBe("ready"); + if (layout.status === "unavailable") { + throw new Error("Expected an authenticated CTE layout"); + } + expect( + resolveAuthenticatedSqlCteEntrypoints( + layout, + source, + slot, + descriptorOnlyDialect, + ), + ).toEqual(layout.mainQueryEntrypoints); + expect(rawRead).toBe(false); + }); + it("projects only proven frame phases and validates positions", () => { const text = "WITH a AS (SELECT 1) SELECT * FROM a"; diff --git a/src/vnext/__tests__/incremental-statement-index.test.ts b/src/vnext/__tests__/incremental-statement-index.test.ts index a0dbfae..4e7e879 100644 --- a/src/vnext/__tests__/incremental-statement-index.test.ts +++ b/src/vnext/__tests__/incremental-statement-index.test.ts @@ -4,6 +4,7 @@ import { buildSqlStatementIndex, DREMIO_SQL_LEXICAL_PROFILE, DUCKDB_SQL_LEXICAL_PROFILE, + isExactSqlStatementSlotSnapshotFor, MAX_SQL_STATEMENT_SLOTS, POSTGRESQL_SQL_LEXICAL_PROFILE, type SqlLexicalProfile, @@ -358,18 +359,93 @@ describe("incremental statement index reuse", () => { }, ); + it("rebuilds when empty change metadata does not match the text", () => { + const previousIndex = buildSqlStatementIndex( + "SELECT 1", + POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + const nextIndex = updateSqlStatementIndex( + previousIndex, + "SELECT 2", + [], + POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + expect(nextIndex).not.toBe(previousIndex); + expect(nextIndex).toEqual( + buildSqlStatementIndex( + "SELECT 2", + POSTGRESQL_SQL_LEXICAL_PROFILE, + ), + ); + }); + + it("rebuilds when the lexical profile changes", () => { + const text = "SELECT 1"; + const previousIndex = buildSqlStatementIndex( + text, + DUCKDB_SQL_LEXICAL_PROFILE, + ); + const nextIndex = updateSqlStatementIndex( + previousIndex, + text, + [], + POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + expect(nextIndex).not.toBe(previousIndex); + expect(nextIndex).toEqual( + buildSqlStatementIndex( + text, + POSTGRESQL_SQL_LEXICAL_PROFILE, + ), + ); + }); + + it("rebuilds when one lexical profile object mutates", () => { + const profile = { ...POSTGRESQL_SQL_LEXICAL_PROFILE }; + const text = "SELECT # ;\n SELECT 2"; + const previousIndex = buildSqlStatementIndex(text, profile); + expect(previousIndex.slots).toHaveLength(2); + profile.hashLineComments = true; + const nextIndex = updateSqlStatementIndex( + previousIndex, + text, + [], + profile, + ); + expect(nextIndex).not.toBe(previousIndex); + expect(nextIndex).toEqual(buildSqlStatementIndex(text, profile)); + expect(nextIndex.slots).toHaveLength(1); + }); + it("reuses unchanged prefix and zero-delta suffix slot identities", () => { const profile = DUCKDB_SQL_LEXICAL_PROFILE; const text = "SELECT 1; SELECT 2; SELECT 3"; const change = replaceFirst(text, "2", "9"); - const { nextIndex, previousIndex } = expectIncrementalMatchesOracle( - text, - [change], - profile, - ); - expect(expectSlot(nextIndex.slots, 0)).toBe( - expectSlot(previousIndex.slots, 0), - ); + const { nextIndex, nextText, previousIndex } = + expectIncrementalMatchesOracle( + text, + [change], + profile, + ); + const sharedPrefix = expectSlot(nextIndex.slots, 0); + expect(sharedPrefix).toBe(expectSlot(previousIndex.slots, 0)); + expect(sharedPrefix.boundaryQuality).toBe("exact"); + expect( + isExactSqlStatementSlotSnapshotFor( + previousIndex, + sharedPrefix, + text, + profile, + ), + ).toBe(true); + expect( + isExactSqlStatementSlotSnapshotFor( + nextIndex, + sharedPrefix, + nextText, + profile, + ), + ).toBe(true); expect(expectSlot(nextIndex.slots, 1)).not.toBe( expectSlot(previousIndex.slots, 1), ); diff --git a/src/vnext/__tests__/query-site.bench.ts b/src/vnext/__tests__/query-site.bench.ts index ca19f79..6a8193d 100644 --- a/src/vnext/__tests__/query-site.bench.ts +++ b/src/vnext/__tests__/query-site.bench.ts @@ -1,8 +1,15 @@ import { bench, describe } from "vitest"; +import { + analyzeSqlCteLayout, + MAX_CTE_DECLARATIONS, +} from "../cte-layout.js"; import { recognizeSqlRelationQuerySite, } from "../query-site.js"; import { DUCKDB_SQL_RELATION_DIALECT } from "../relation-dialect.js"; +import { + recognizeSqlRelationQuerySiteWithCteLayout, +} from "../relation-query-site.js"; import { createIdentitySqlSource } from "../source.js"; import { buildSqlStatementIndex, @@ -61,6 +68,103 @@ const usingHeavySlot = findSqlStatementSlot( usingHeavyPosition, "left", ); +const cteQueryPrefix = + "WITH cte_name AS (SELECT 1) SELECT "; +const cteQuerySuffix = " FROM schema_prefix"; +const cteProjectedListLength = + TEN_KIBIBYTES - + cteQueryPrefix.length - + cteQuerySuffix.length; +const cteProjectedList = `${"x,".repeat( + Math.floor((cteProjectedListLength - 1) / 2), +)}x`; +const tenKilobyteCteQuery = + `${cteQueryPrefix}${cteProjectedList}` + + `${" ".repeat( + cteProjectedListLength - cteProjectedList.length, + )}${cteQuerySuffix}`; +if (tenKilobyteCteQuery.length !== TEN_KIBIBYTES) { + throw new Error("CTE query benchmark fixture must be exactly 10 KiB"); +} +const cteSource = createIdentitySqlSource(tenKilobyteCteQuery); +const cteIndex = buildSqlStatementIndex( + cteSource.analysisText, + dialect.lexicalProfile, +); +const ctePosition = tenKilobyteCteQuery.length; +const cteSlot = findSqlStatementSlot( + cteIndex, + ctePosition, + "left", +); +if (cteSlot.boundaryQuality === "opaque") { + throw new Error("CTE query benchmark requires an exact statement"); +} +const maximumCteQuery = + `WITH ${Array.from( + { length: MAX_CTE_DECLARATIONS }, + (_, index) => `cte_${index} AS (SELECT ${index})`, + ).join(", ")} SELECT * FROM target`; +const maximumCteSource = + createIdentitySqlSource(maximumCteQuery); +const maximumCteIndex = buildSqlStatementIndex( + maximumCteSource.analysisText, + dialect.lexicalProfile, +); +const maximumCtePosition = maximumCteQuery.length; +const maximumCteSlot = findSqlStatementSlot( + maximumCteIndex, + maximumCtePosition, + "left", +); +if (maximumCteSlot.boundaryQuality === "opaque") { + throw new Error("Maximum CTE benchmark requires an exact statement"); +} +const ctePreflightLayout = analyzeSqlCteLayout( + cteSource, + cteIndex, + cteSlot, + DUCKDB_SQL_RELATION_DIALECT.cteLayout, +); +const ctePreflightResult = + recognizeSqlRelationQuerySiteWithCteLayout( + cteSource, + cteSlot, + ctePosition, + DUCKDB_SQL_RELATION_DIALECT, + ctePreflightLayout, + ); +if ( + ctePreflightLayout.status !== "ready" || + ctePreflightResult.status !== "ready" || + ctePreflightResult.anchor !== "from" +) { + throw new Error("CTE query benchmark must exercise a ready FROM site"); +} +const maximumCtePreflightLayout = analyzeSqlCteLayout( + maximumCteSource, + maximumCteIndex, + maximumCteSlot, + DUCKDB_SQL_RELATION_DIALECT.cteLayout, +); +const maximumCtePreflightResult = + recognizeSqlRelationQuerySiteWithCteLayout( + maximumCteSource, + maximumCteSlot, + maximumCtePosition, + DUCKDB_SQL_RELATION_DIALECT, + maximumCtePreflightLayout, + ); +if ( + maximumCtePreflightLayout.status !== "ready" || + maximumCtePreflightLayout.declarations.length !== + MAX_CTE_DECLARATIONS || + maximumCtePreflightLayout.mainQueryEntrypoints.length !== 1 || + maximumCtePreflightResult.status !== "ready" || + maximumCtePreflightResult.prefix.value !== "target" +) { + throw new Error("Maximum CTE benchmark must exercise all declarations"); +} describe("query-site recognizer", () => { bench("10 KiB active statement", () => { @@ -84,4 +188,36 @@ describe("query-site recognizer", () => { dialect, ); }); + + bench("10 KiB WITH main-query double scan", () => { + const layout = analyzeSqlCteLayout( + cteSource, + cteIndex, + cteSlot, + DUCKDB_SQL_RELATION_DIALECT.cteLayout, + ); + recognizeSqlRelationQuerySiteWithCteLayout( + cteSource, + cteSlot, + ctePosition, + DUCKDB_SQL_RELATION_DIALECT, + layout, + ); + }); + + bench("256 CTE main-query double scan", () => { + const layout = analyzeSqlCteLayout( + maximumCteSource, + maximumCteIndex, + maximumCteSlot, + DUCKDB_SQL_RELATION_DIALECT.cteLayout, + ); + recognizeSqlRelationQuerySiteWithCteLayout( + maximumCteSource, + maximumCteSlot, + maximumCtePosition, + DUCKDB_SQL_RELATION_DIALECT, + layout, + ); + }); }); diff --git a/src/vnext/__tests__/query-site.test.ts b/src/vnext/__tests__/query-site.test.ts index 10f268b..089b52a 100644 --- a/src/vnext/__tests__/query-site.test.ts +++ b/src/vnext/__tests__/query-site.test.ts @@ -1,4 +1,8 @@ import { describe, expect, it } from "vitest"; +import { + analyzeSqlCteLayout, + type SqlCteLayout, +} from "../cte-layout.js"; import { MAX_QUERY_SITE_DEPTH, MAX_QUERY_SITE_IDENTIFIER_LENGTH, @@ -10,10 +14,15 @@ import { type SqlQuerySiteDialect, type SqlQuerySiteResult, } from "../query-site.js"; +import { + recognizeSqlRelationQuerySiteWithCteLayout, +} from "../relation-query-site.js"; import { BIGQUERY_SQL_RELATION_DIALECT, + DREMIO_SQL_RELATION_DIALECT, DUCKDB_SQL_RELATION_DIALECT, POSTGRESQL_SQL_RELATION_DIALECT, + type SqlRelationDialectRuntime, } from "../relation-dialect.js"; import { createIdentitySqlSource, @@ -24,6 +33,8 @@ import { buildSqlStatementIndex, findSqlStatementSlot, POSTGRESQL_SQL_LEXICAL_PROFILE, + type ExactSqlStatementSlot, + updateSqlStatementIndex, } from "../statement-index.js"; const postgresDialect = POSTGRESQL_SQL_RELATION_DIALECT.querySite; @@ -84,6 +95,54 @@ function expectReady( return result; } +function cteFixture( + marked: string, + dialect: SqlRelationDialectRuntime = + POSTGRESQL_SQL_RELATION_DIALECT, +): { + readonly layout: Exclude< + SqlCteLayout, + { readonly status: "unavailable" } + >; + readonly position: number; + readonly result: SqlQuerySiteResult; + readonly slot: ExactSqlStatementSlot; + readonly source: SqlSourceSnapshot; +} { + const { position, text } = markedSource(marked); + const source = createIdentitySqlSource(text); + const index = buildSqlStatementIndex( + source.analysisText, + dialect.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot(index, position, "left"); + if (slot.boundaryQuality === "opaque") { + throw new Error("CTE query fixture requires an exact statement"); + } + const layout = analyzeSqlCteLayout( + source, + index, + slot, + dialect.cteLayout, + ); + if (layout.status === "unavailable") { + throw new Error("CTE query fixture requires an available layout"); + } + return { + layout, + position, + result: recognizeSqlRelationQuerySiteWithCteLayout( + source, + slot, + position, + dialect, + layout, + ), + slot, + source, + }; +} + describe("partial SELECT relation query sites", () => { it.each([ ["SELECT * FROM |", "from"], @@ -306,6 +365,573 @@ describe("partial SELECT relation query sites", () => { }); }); +describe("authenticated CTE main-query entrypoints", () => { + it.each([ + [ + "WITH cte_name AS (SELECT 1) SELECT * FROM |", + POSTGRESQL_SQL_RELATION_DIALECT, + ], + [ + "WITH cte_name AS (SELECT 1) SELECT * FROM |", + DUCKDB_SQL_RELATION_DIALECT, + ], + [ + "WITH cte_name AS (SELECT 1) SELECT * FROM |", + BIGQUERY_SQL_RELATION_DIALECT, + ], + [ + "WITH cte_name(id) AS (SELECT 1) SELECT * FROM |", + DREMIO_SQL_RELATION_DIALECT, + ], + ] as const)("recognizes a built-in main query in %s", (marked, dialect) => { + const result = expectReady(cteFixture(marked, dialect).result); + expect(result.anchor).toBe("from"); + expect(result.recognition).toEqual({ + issues: [], + quality: "exact", + }); + }); + + it("recognizes nested CTE main queries by exact offset and depth", () => { + const result = expectReady( + cteFixture( + "WITH outer_cte AS (WITH inner_cte AS (SELECT 1) SELECT * FROM |) SELECT 1", + ).result, + ); + expect(result.anchor).toBe("from"); + }); + + it("returns to the enclosing entrypoint after a nested CTE closes", () => { + const result = expectReady( + cteFixture( + "WITH outer_cte AS (WITH inner_cte AS (SELECT 1) SELECT * FROM inner_cte) SELECT * FROM |", + ).result, + ); + expect(result.anchor).toBe("from"); + expect(result.recognition.quality).toBe("exact"); + }); + + it("keeps ordinary CTE body recognition independent of entrypoints", () => { + const marked = + "WITH local AS (SELECT * FROM |) SELECT * FROM local"; + expect(recognize(marked).status).toBe("ready"); + expect(cteFixture(marked).result.status).toBe("ready"); + }); + + it("translates statement-relative entrypoints for later statements", () => { + const result = expectReady( + cteFixture( + "SELECT 1; WITH local AS (SELECT 1) SELECT * FROM schema.ta|", + ).result, + ); + expect(result.prefix).toEqual({ quoted: false, value: "ta" }); + expect(result.typedPathRange).toMatchObject({ + from: 40, + to: 49, + }); + }); + + it("rejects copied, proxied, raw, cross-source, and cross-dialect evidence", () => { + const fixture = cteFixture( + "WITH local AS (SELECT 1) SELECT * FROM |", + ); + const copied = { ...fixture.layout } as SqlCteLayout; + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + fixture.source, + fixture.slot, + fixture.position, + POSTGRESQL_SQL_RELATION_DIALECT, + copied, + ), + ).toEqual({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + + let invoked = false; + const proxied = new Proxy(fixture.layout, { + get() { + invoked = true; + throw new Error("hostile"); + }, + getOwnPropertyDescriptor() { + invoked = true; + throw new Error("hostile"); + }, + ownKeys() { + invoked = true; + throw new Error("hostile"); + }, + }); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + fixture.source, + fixture.slot, + fixture.position, + POSTGRESQL_SQL_RELATION_DIALECT, + proxied, + ).status, + ).toBe("unavailable"); + expect(invoked).toBe(false); + + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + fixture.source, + fixture.slot, + fixture.position, + POSTGRESQL_SQL_RELATION_DIALECT, + fixture.layout.mainQueryEntrypoints as never, + ).status, + ).toBe("unavailable"); + + const secondSource = createIdentitySqlSource( + fixture.source.originalText, + ); + const secondIndex = buildSqlStatementIndex( + secondSource.analysisText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const secondSlot = findSqlStatementSlot( + secondIndex, + fixture.position, + "left", + ); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + secondSource, + secondSlot, + fixture.position, + POSTGRESQL_SQL_RELATION_DIALECT, + fixture.layout, + ).status, + ).toBe("unavailable"); + + const mixedRuntime: SqlRelationDialectRuntime = { + ...POSTGRESQL_SQL_RELATION_DIALECT, + querySite: DUCKDB_SQL_RELATION_DIALECT.querySite, + }; + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + fixture.source, + fixture.slot, + fixture.position, + mixedRuntime, + fixture.layout, + ).status, + ).toBe("unavailable"); + + let runtimeInvoked = false; + const proxiedRuntime = new Proxy( + POSTGRESQL_SQL_RELATION_DIALECT, + { + get() { + runtimeInvoked = true; + throw new Error("hostile"); + }, + getOwnPropertyDescriptor() { + runtimeInvoked = true; + throw new Error("hostile"); + }, + ownKeys() { + runtimeInvoked = true; + throw new Error("hostile"); + }, + }, + ); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + fixture.source, + fixture.slot, + fixture.position, + proxiedRuntime, + fixture.layout, + ).status, + ).toBe("unavailable"); + expect(runtimeInvoked).toBe(false); + + let slotInvoked = false; + const proxiedSlot = new Proxy(fixture.slot, { + get() { + slotInvoked = true; + throw new Error("hostile"); + }, + getOwnPropertyDescriptor() { + slotInvoked = true; + throw new Error("hostile"); + }, + ownKeys() { + slotInvoked = true; + throw new Error("hostile"); + }, + }); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + fixture.source, + proxiedSlot, + fixture.position, + POSTGRESQL_SQL_RELATION_DIALECT, + fixture.layout, + ).status, + ).toBe("unavailable"); + expect(slotInvoked).toBe(false); + + const rebuiltIndex = buildSqlStatementIndex( + fixture.source.analysisText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const rebuiltSlot = findSqlStatementSlot( + rebuiltIndex, + fixture.position, + "left", + ); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + fixture.source, + rebuiltSlot, + fixture.position, + POSTGRESQL_SQL_RELATION_DIALECT, + fixture.layout, + ).status, + ).toBe("unavailable"); + const mismatchedLayout = analyzeSqlCteLayout( + fixture.source, + rebuiltIndex, + fixture.slot, + POSTGRESQL_SQL_RELATION_DIALECT.cteLayout, + ); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + fixture.source, + fixture.slot, + fixture.position, + POSTGRESQL_SQL_RELATION_DIALECT, + mismatchedLayout, + ).status, + ).toBe("unavailable"); + + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + fixture.source, + fixture.slot, + fixture.position, + POSTGRESQL_SQL_RELATION_DIALECT, + null as never, + ).status, + ).toBe("unavailable"); + + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + fixture.source, + fixture.slot, + fixture.position, + DUCKDB_SQL_RELATION_DIALECT, + fixture.layout, + ).status, + ).toBe("unavailable"); + }); + + it("does not authenticate mutable statement slots", () => { + const first = + "WITH cte_name AS (SELECT 1) SELECT * FROM target"; + const second = + "XXXX cte_name AS (SELECT 1) SELECT * FROM target"; + const source = createIdentitySqlSource(`${first};${second}`); + const index = buildSqlStatementIndex( + source.analysisText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const firstSlot = findSqlStatementSlot(index, first.length, "left"); + const secondSlot = findSqlStatementSlot( + index, + source.analysisText.length, + "left", + ); + if ( + firstSlot.boundaryQuality === "opaque" || + secondSlot.boundaryQuality === "opaque" + ) { + throw new Error("Mutable slot fixture requires exact statements"); + } + const mutableSlot: ExactSqlStatementSlot = { + ...firstSlot, + extent: { ...firstSlot.extent }, + source: { ...firstSlot.source }, + terminator: firstSlot.terminator + ? { ...firstSlot.terminator } + : null, + }; + const layout = analyzeSqlCteLayout( + source, + index, + mutableSlot, + POSTGRESQL_SQL_RELATION_DIALECT.cteLayout, + ); + Object.assign(mutableSlot, { + endState: secondSlot.endState, + extent: { ...secondSlot.extent }, + hasCode: secondSlot.hasCode, + source: { ...secondSlot.source }, + terminator: secondSlot.terminator, + }); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + source, + mutableSlot, + source.analysisText.length, + POSTGRESQL_SQL_RELATION_DIALECT, + layout, + ), + ).toEqual({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + }); + + it("binds authentic slots to their analysis text", () => { + const source = createIdentitySqlSource( + "DELIMITER $$;SELECT * FROM ", + ); + const foreignText = "xxxxxxxxxxxx;SELECT * FROM "; + expect(foreignText.length).toBe(source.analysisText.length); + const foreignIndex = buildSqlStatementIndex( + foreignText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const foreignSlot = findSqlStatementSlot( + foreignIndex, + foreignText.length, + "left", + ); + if (foreignSlot.boundaryQuality === "opaque") { + throw new Error("Foreign slot fixture requires an exact statement"); + } + const layout = analyzeSqlCteLayout( + source, + foreignIndex, + foreignSlot, + POSTGRESQL_SQL_RELATION_DIALECT.cteLayout, + ); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + source, + foreignSlot, + source.analysisText.length, + POSTGRESQL_SQL_RELATION_DIALECT, + layout, + ), + ).toEqual({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + }); + + it("binds authentic slots to their lexical profile", () => { + const source = createIdentitySqlSource("SELECT * FROM "); + const index = buildSqlStatementIndex( + source.analysisText, + DUCKDB_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot( + index, + source.analysisText.length, + "left", + ); + if (slot.boundaryQuality === "opaque") { + throw new Error("Wrong-profile fixture requires an exact statement"); + } + const layout = analyzeSqlCteLayout( + source, + index, + slot, + POSTGRESQL_SQL_RELATION_DIALECT.cteLayout, + ); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + source, + slot, + source.analysisText.length, + POSTGRESQL_SQL_RELATION_DIALECT, + layout, + ), + ).toEqual({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + }); + + it("keeps old and new incremental statement contexts valid", () => { + const first = + "WITH cte_name AS (SELECT 1) SELECT * FROM target"; + const oldText = `${first}; SELECT 2; SELECT 3`; + const oldSource = createIdentitySqlSource(oldText); + const profile = + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile; + const oldIndex = buildSqlStatementIndex(oldText, profile); + const oldSlot = findSqlStatementSlot( + oldIndex, + first.length, + "left", + ); + if (oldSlot.boundaryQuality === "opaque") { + throw new Error("Incremental fixture requires exact statements"); + } + const oldLayout = analyzeSqlCteLayout( + oldSource, + oldIndex, + oldSlot, + POSTGRESQL_SQL_RELATION_DIALECT.cteLayout, + ); + const editFrom = oldText.indexOf("SELECT 2") + "SELECT ".length; + const newText = `${oldText.slice(0, editFrom)}4${oldText.slice( + editFrom + 1, + )}`; + const newIndex = updateSqlStatementIndex( + oldIndex, + newText, + [{ from: editFrom, insert: "4", to: editFrom + 1 }], + profile, + ); + const newSource = createIdentitySqlSource(newText); + const newSlot = findSqlStatementSlot( + newIndex, + first.length, + "left", + ); + if (newSlot.boundaryQuality === "opaque") { + throw new Error("Incremental fixture requires exact statements"); + } + expect(newSlot).toBe(oldSlot); + const newLayout = analyzeSqlCteLayout( + newSource, + newIndex, + newSlot, + POSTGRESQL_SQL_RELATION_DIALECT.cteLayout, + ); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + oldSource, + oldSlot, + first.length, + POSTGRESQL_SQL_RELATION_DIALECT, + oldLayout, + ).status, + ).toBe("ready"); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + newSource, + newSlot, + first.length, + POSTGRESQL_SQL_RELATION_DIALECT, + newLayout, + ).status, + ).toBe("ready"); + }); + + it("preserves opaque statement failures for authentic slots", () => { + const source = createIdentitySqlSource("DELIMITER $$"); + const index = buildSqlStatementIndex( + source.analysisText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot( + index, + source.analysisText.length, + "left", + ); + const fixture = cteFixture( + "WITH local AS (SELECT 1) SELECT * FROM |", + ); + expect(slot.boundaryQuality).toBe("opaque"); + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + source, + slot, + source.analysisText.length, + POSTGRESQL_SQL_RELATION_DIALECT, + fixture.layout, + ), + ).toEqual({ + reason: "opaque-statement", + status: "unavailable", + }); + }); + + it.each([ + ["invalid-header", "header"], + ["skipped", "skipped"], + ["wrong-depth", "depth"], + ["wrong-token", "token"], + ] as const)( + "fails closed when authenticated source evidence becomes %s", + (_, mutation) => { + const text = + "WITH cte_name AS (SELECT 1) SELECT * FROM target"; + const mutableSource: SqlSourceSnapshot = { + ...createIdentitySqlSource(text), + }; + const position = text.length; + const index = buildSqlStatementIndex( + mutableSource.analysisText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot(index, position, "left"); + if (slot.boundaryQuality === "opaque") { + throw new Error("Mutable fixture requires an exact statement"); + } + const layout = analyzeSqlCteLayout( + mutableSource, + index, + slot, + POSTGRESQL_SQL_RELATION_DIALECT.cteLayout, + ); + const mainQueryStart = text.lastIndexOf("SELECT"); + const nextAnalysisText = + mutation === "header" + ? `XXXX${text.slice(4)}` + : mutation === "depth" + ? `${text.slice(0, mainQueryStart - 2)} ${text.slice( + mainQueryStart - 1, + )}` + : `${text.slice(0, mainQueryStart)}${ + mutation === "skipped" ? "/*x*/ " : "DELETE" + }${text.slice(mainQueryStart + 6)}`; + (mutableSource as { analysisText: string }).analysisText = + nextAnalysisText; + expect( + recognizeSqlRelationQuerySiteWithCteLayout( + mutableSource, + slot, + position, + POSTGRESQL_SQL_RELATION_DIALECT, + layout, + ), + ).toEqual({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + }, + ); + + it("does not make raw WITH statements query candidates", () => { + const marked = + "WITH local AS (SELECT 1) SELECT * FROM |"; + expect(recognize(marked)).toEqual({ + reason: "not-relation-position", + status: "inactive", + }); + }); + + it("does not invent entrypoints for incomplete CTE headers", () => { + const fixture = cteFixture( + "WITH cte_name AS (SELECT 1), SELECT * FROM |", + ); + expect(fixture.layout.status).toBe("partial"); + expect(fixture.layout.mainQueryEntrypoints).toEqual([]); + expect(fixture.result.status).not.toBe("ready"); + }); +}); + describe("fail-closed query-site behavior", () => { it.each([ ["SELECT x IS DISTINCT FROM |", "inactive"], diff --git a/src/vnext/__tests__/relation-dialect.test.ts b/src/vnext/__tests__/relation-dialect.test.ts index 8dca8d5..38a4c9f 100644 --- a/src/vnext/__tests__/relation-dialect.test.ts +++ b/src/vnext/__tests__/relation-dialect.test.ts @@ -10,6 +10,9 @@ import { POSTGRESQL_SQL_RELATION_DIALECT, type SqlRelationDialectRuntime, } from "../relation-dialect.js"; +import { + isSqlRelationDialectRuntime, +} from "../relation-runtime-auth.js"; import type { SqlCanonicalRelationPath, SqlIdentifierDecodeResult, @@ -64,15 +67,21 @@ function analyze( text: string, ): SqlCteLayout { const source = createIdentitySqlSource(text); - const slot = buildSqlStatementIndex( + const index = buildSqlStatementIndex( source.analysisText, runtime.querySite.lexicalProfile, - ).slots[0]; + ); + const slot = index.slots[0]; expect(slot?.boundaryQuality).toBe("exact"); if (!slot || slot.boundaryQuality !== "exact") { throw new Error("Expected one exact SQL statement"); } - return analyzeSqlCteLayout(source, slot, runtime.cteLayout); + return analyzeSqlCteLayout( + source, + index, + slot, + runtime.cteLayout, + ); } function expectDeepFrozenRuntime( @@ -86,6 +95,20 @@ function expectDeepFrozenRuntime( } describe("built-in relation dialect runtime", () => { + it("authenticates only package-owned coherent runtime aggregates", () => { + expect( + isSqlRelationDialectRuntime( + POSTGRESQL_SQL_RELATION_DIALECT, + ), + ).toBe(true); + expect( + isSqlRelationDialectRuntime({ + ...POSTGRESQL_SQL_RELATION_DIALECT, + }), + ).toBe(false); + expect(isSqlRelationDialectRuntime(null)).toBe(false); + }); + it("owns stable, deeply frozen, coherent views", () => { for (const runtime of Object.values(RUNTIMES)) { expectDeepFrozenRuntime(runtime); diff --git a/src/vnext/__tests__/source.test.ts b/src/vnext/__tests__/source.test.ts index c98bf7e..62178d9 100644 --- a/src/vnext/__tests__/source.test.ts +++ b/src/vnext/__tests__/source.test.ts @@ -3,6 +3,7 @@ import { createIdentitySqlSource, createMaskedSqlSource, findSqlEmbeddedRegionAtOrAfter, + isSqlSourceSnapshot, mapAnalysisRangeToOriginal, mapOriginalRangeToAnalysis, MAX_SQL_EMBEDDED_REGIONS, @@ -147,6 +148,9 @@ describe("SQL source snapshots", () => { expect(source.embeddedRegions).toEqual([]); expect(Object.isFrozen(source)).toBe(true); expect(Object.isFrozen(source.embeddedRegions)).toBe(true); + expect(isSqlSourceSnapshot(source)).toBe(true); + expect(isSqlSourceSnapshot({ ...source })).toBe(false); + expect(isSqlSourceSnapshot(null)).toBe(false); }); it("bounds and validates source text", () => { diff --git a/src/vnext/__tests__/statement-index.test.ts b/src/vnext/__tests__/statement-index.test.ts index 0a17bd1..26bc41f 100644 --- a/src/vnext/__tests__/statement-index.test.ts +++ b/src/vnext/__tests__/statement-index.test.ts @@ -5,6 +5,9 @@ import { DREMIO_SQL_LEXICAL_PROFILE, DUCKDB_SQL_LEXICAL_PROFILE, findSqlStatementSlot, + isExactSqlStatementSlotSnapshot, + isExactSqlStatementSlotSnapshotFor, + isSqlStatementSlotSnapshot, MAX_SQL_STATEMENT_SLOTS, POSTGRESQL_SQL_LEXICAL_PROFILE, type SqlLexicalProfile, @@ -78,6 +81,47 @@ function expectPartition(text: string, index: SqlStatementIndex): void { } describe("statement partition", () => { + it("authenticates only package-created immutable slot snapshots", () => { + const exactIndex = buildSqlStatementIndex( + "SELECT 1", + POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + const exact = itemAt( + exactIndex.slots, + 0, + ); + const opaqueIndex = buildSqlStatementIndex( + "DELIMITER $$", + POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + const opaque = itemAt( + opaqueIndex.slots, + 0, + ); + expect(isSqlStatementSlotSnapshot(exact)).toBe(true); + expect(isExactSqlStatementSlotSnapshot(exact)).toBe(true); + expect( + isExactSqlStatementSlotSnapshotFor( + exactIndex, + exact, + "SELECT 1", + POSTGRESQL_SQL_LEXICAL_PROFILE, + ), + ).toBe(true); + expect( + isExactSqlStatementSlotSnapshotFor( + exactIndex, + exact, + "SELECT 2", + POSTGRESQL_SQL_LEXICAL_PROFILE, + ), + ).toBe(false); + expect(isSqlStatementSlotSnapshot(opaque)).toBe(true); + expect(isExactSqlStatementSlotSnapshot(opaque)).toBe(false); + expect(isSqlStatementSlotSnapshot({ ...exact })).toBe(false); + expect(isExactSqlStatementSlotSnapshot(null)).toBe(false); + }); + it.each([ [ "", diff --git a/src/vnext/cte-layout.ts b/src/vnext/cte-layout.ts index d5c9bb5..820fefc 100644 --- a/src/vnext/cte-layout.ts +++ b/src/vnext/cte-layout.ts @@ -4,8 +4,15 @@ import { type BoundedSqlLexerResource, } from "./bounded-sql-lexer.js"; import type { SqlLexicalProfile } from "./lexical.js"; -import type { SqlSourceSnapshot } from "./source.js"; -import type { ExactSqlStatementSlot } from "./statement-index.js"; +import { + isSqlSourceSnapshot, + type SqlSourceSnapshot, +} from "./source.js"; +import { + isExactSqlStatementSlotSnapshotFor, + type ExactSqlStatementSlot, + type SqlStatementIndex, +} from "./statement-index.js"; import type { SqlIdentifierComponent } from "./types.js"; const cteRangeBrand: unique symbol = Symbol("SqlCteRange"); @@ -166,6 +173,20 @@ export interface SqlCteVisibility { readonly shadowing: SqlCteShadowing; } +interface SqlCteLayoutProvenance { + readonly dialect: SqlCteLayoutDialect; + readonly entrypoints: readonly SqlCteMainQueryEntrypoint[]; + readonly index: SqlStatementIndex; + readonly lexicalProfile: SqlLexicalProfile; + readonly slot: ExactSqlStatementSlot; + readonly source: SqlSourceSnapshot; +} + +const sqlCteLayoutProvenance = new WeakMap< + object, + SqlCteLayoutProvenance +>(); + type HeaderState = | "after-as" | "after-body" @@ -872,7 +893,7 @@ function freezeLayout( exactThrough: number, issues: Set, resource?: SqlCteLayoutResource, -): SqlCteLayout { +): Exclude { const unknownClasses = (identityIndex: number): readonly number[] => Object.freeze( [ @@ -1012,6 +1033,7 @@ function freezeLayout( export function analyzeSqlCteLayout( source: SqlSourceSnapshot, + index: SqlStatementIndex, slot: ExactSqlStatementSlot, dialect: SqlCteLayoutDialect, ): SqlCteLayout { @@ -1551,7 +1573,7 @@ export function analyzeSqlCteLayout( relations, exactThrough, ); - return freezeLayout( + const layout = freezeLayout( frames, declarations, draftDeclarations, @@ -1561,6 +1583,58 @@ export function analyzeSqlCteLayout( issues, resource, ); + if ( + isSqlSourceSnapshot(source) && + isExactSqlStatementSlotSnapshotFor( + index, + slot, + source.analysisText, + lexicalProfile, + ) + ) { + sqlCteLayoutProvenance.set( + layout, + Object.freeze({ + dialect, + entrypoints: layout.mainQueryEntrypoints, + index, + lexicalProfile, + slot, + source, + }), + ); + } + return layout; +} + +export function resolveAuthenticatedSqlCteEntrypoints( + candidate: unknown, + source: SqlSourceSnapshot, + slot: ExactSqlStatementSlot, + dialect: SqlCteLayoutDialect, +): readonly SqlCteMainQueryEntrypoint[] | null { + if ( + candidate === null || + typeof candidate !== "object" || + !isSqlSourceSnapshot(source) + ) { + return null; + } + const provenance = sqlCteLayoutProvenance.get(candidate); + if ( + provenance?.source !== source || + provenance.slot !== slot || + provenance.dialect !== dialect || + !isExactSqlStatementSlotSnapshotFor( + provenance.index, + slot, + source.analysisText, + provenance.lexicalProfile, + ) + ) { + return null; + } + return provenance.entrypoints; } function framePhase( diff --git a/src/vnext/query-site.ts b/src/vnext/query-site.ts index f08321e..33c9480 100644 --- a/src/vnext/query-site.ts +++ b/src/vnext/query-site.ts @@ -32,6 +32,11 @@ export interface SqlQuerySiteRange { readonly to: number; } +export interface SqlQuerySiteEntrypoint { + readonly depth: number; + readonly from: number; +} + export type SqlQuerySiteResource = | "active-statement" | "identifier-path" @@ -1068,11 +1073,14 @@ function resultAtGap( return inactive("not-relation-position"); } -export function recognizeSqlRelationQuerySite( +function recognizeSqlRelationQuerySiteInternal( source: SqlSourceSnapshot, slot: SqlStatementSlot, position: number, dialect: SqlQuerySiteDialect, + authenticatedEntrypoints: + | readonly SqlQuerySiteEntrypoint[] + | null, ): SqlQuerySiteResult { if (slot.boundaryQuality === "opaque") { return unavailable("opaque-statement"); @@ -1120,6 +1128,7 @@ export function recognizeSqlRelationQuerySite( ); const frames: QueryFrame[] = []; const queryCandidates = new Set([0]); + let entrypointIndex = 0; let depth = 0; let sawSelect = false; let statementTainted = false; @@ -1422,7 +1431,20 @@ export function recognizeSqlRelationQuerySite( if (token.kind === "word") { const isSelect = wordEquals(source.analysisText, token, "select"); - if (queryCandidates.has(depth)) { + const entrypoint = + authenticatedEntrypoints?.[entrypointIndex]; + const relativeTokenFrom = token.from - slot.source.from; + const isAuthenticatedEntrypoint = + entrypoint?.from === relativeTokenFrom && + entrypoint.depth === depth && + isSelect; + if (isAuthenticatedEntrypoint) { + entrypointIndex += 1; + } + if ( + queryCandidates.has(depth) || + isAuthenticatedEntrypoint + ) { queryCandidates.delete(depth); if ( isSelect && @@ -1525,3 +1547,34 @@ export function recognizeSqlRelationQuerySite( } } } + +export function recognizeSqlRelationQuerySite( + source: SqlSourceSnapshot, + slot: SqlStatementSlot, + position: number, + dialect: SqlQuerySiteDialect, +): SqlQuerySiteResult { + return recognizeSqlRelationQuerySiteInternal( + source, + slot, + position, + dialect, + null, + ); +} + +export function recognizeSqlRelationQuerySiteWithEntrypoints( + source: SqlSourceSnapshot, + slot: ExactSqlStatementSlot, + position: number, + dialect: SqlQuerySiteDialect, + entrypoints: readonly SqlQuerySiteEntrypoint[], +): SqlQuerySiteResult { + return recognizeSqlRelationQuerySiteInternal( + source, + slot, + position, + dialect, + entrypoints, + ); +} diff --git a/src/vnext/relation-dialect.ts b/src/vnext/relation-dialect.ts index fe01d17..76bbf50 100644 --- a/src/vnext/relation-dialect.ts +++ b/src/vnext/relation-dialect.ts @@ -20,6 +20,9 @@ import { type SqlQuerySiteDialect, } from "./query-site.js"; import { isSqlRelationReservedWord } from "./relation-reserved-words.js"; +import { + registerSqlRelationDialectRuntime, +} from "./relation-runtime-auth.js"; import type { SqlCteIdentifierComparison, SqlCteIdentifierPrefixMatch, @@ -1413,7 +1416,13 @@ function createRuntime(spec: RelationDialectSpec): SqlRelationDialectRuntime { maximumPathDepth: spec.maximumPathDepth, supportsNaturalJoin: spec.supportsNaturalJoin, }); - return Object.freeze({ completion, cteLayout, querySite }); + return registerSqlRelationDialectRuntime( + Object.freeze({ + completion, + cteLayout, + querySite, + }), + ); } function createGrammar( diff --git a/src/vnext/relation-query-site.ts b/src/vnext/relation-query-site.ts new file mode 100644 index 0000000..f324acb --- /dev/null +++ b/src/vnext/relation-query-site.ts @@ -0,0 +1,59 @@ +import { + resolveAuthenticatedSqlCteEntrypoints, + type SqlCteLayout, +} from "./cte-layout.js"; +import { + recognizeSqlRelationQuerySiteWithEntrypoints, + type SqlQuerySiteResult, +} from "./query-site.js"; +import type { SqlRelationDialectRuntime } from "./relation-dialect.js"; +import { isSqlRelationDialectRuntime } from "./relation-runtime-auth.js"; +import { + isSqlSourceSnapshot, + type SqlSourceSnapshot, +} from "./source.js"; +import { + isSqlStatementSlotSnapshot, + type SqlStatementSlot, +} from "./statement-index.js"; + +function unavailable( + reason: "ambiguous-query-site" | "opaque-statement", +): SqlQuerySiteResult { + return Object.freeze({ reason, status: "unavailable" }); +} + +export function recognizeSqlRelationQuerySiteWithCteLayout( + source: SqlSourceSnapshot, + slot: SqlStatementSlot, + position: number, + dialect: SqlRelationDialectRuntime, + layout: SqlCteLayout, +): SqlQuerySiteResult { + if ( + !isSqlRelationDialectRuntime(dialect) || + !isSqlSourceSnapshot(source) || + !isSqlStatementSlotSnapshot(slot) + ) { + return unavailable("ambiguous-query-site"); + } + if (slot.boundaryQuality === "opaque") { + return unavailable("opaque-statement"); + } + const entrypoints = resolveAuthenticatedSqlCteEntrypoints( + layout, + source, + slot, + dialect.cteLayout, + ); + if (!entrypoints) { + return unavailable("ambiguous-query-site"); + } + return recognizeSqlRelationQuerySiteWithEntrypoints( + source, + slot, + position, + dialect.querySite, + entrypoints, + ); +} diff --git a/src/vnext/relation-runtime-auth.ts b/src/vnext/relation-runtime-auth.ts new file mode 100644 index 0000000..0bcf73b --- /dev/null +++ b/src/vnext/relation-runtime-auth.ts @@ -0,0 +1,18 @@ +const sqlRelationDialectRuntimes = new WeakSet(); + +export function registerSqlRelationDialectRuntime< + Runtime extends object, +>(runtime: Runtime): Runtime { + sqlRelationDialectRuntimes.add(runtime); + return runtime; +} + +export function isSqlRelationDialectRuntime( + candidate: unknown, +): boolean { + return ( + candidate !== null && + typeof candidate === "object" && + sqlRelationDialectRuntimes.has(candidate) + ); +} diff --git a/src/vnext/source.ts b/src/vnext/source.ts index 5da81ba..24760ef 100644 --- a/src/vnext/source.ts +++ b/src/vnext/source.ts @@ -7,6 +7,7 @@ const MAX_EMBEDDED_LANGUAGE_LENGTH = 256; const MASK_CHUNK_LENGTH = 64 * 1024; const EMPTY_EMBEDDED_REGIONS: readonly SqlEmbeddedRegion[] = Object.freeze([]); const sourceErrors = new WeakSet(); +const sqlSourceSnapshots = new WeakSet(); export type SqlSourceErrorCode = | "invalid-source" @@ -38,6 +39,16 @@ export interface SqlSourceSnapshot { readonly originalText: string; } +export function isSqlSourceSnapshot( + candidate: unknown, +): candidate is SqlSourceSnapshot { + return ( + candidate !== null && + typeof candidate === "object" && + sqlSourceSnapshots.has(candidate) + ); +} + export function findSqlEmbeddedRegionAtOrAfter( source: SqlSourceSnapshot, position: number, @@ -292,11 +303,13 @@ function maskRegion(text: string, from: number, to: number): string { export function createIdentitySqlSource(text: unknown): SqlSourceSnapshot { const originalText = normalizeSourceText(text); - return Object.freeze({ + const source = Object.freeze({ analysisText: originalText, embeddedRegions: EMPTY_EMBEDDED_REGIONS, originalText, }); + sqlSourceSnapshots.add(source); + return source; } export function createMaskedSqlSource( @@ -323,11 +336,13 @@ export function createMaskedSqlSource( } output.push(originalText.slice(cursor)); const analysisText = output.join(""); - return Object.freeze({ + const source = Object.freeze({ analysisText, embeddedRegions, originalText, }); + sqlSourceSnapshots.add(source); + return source; } export function mapAnalysisRangeToOriginal( diff --git a/src/vnext/statement-index.ts b/src/vnext/statement-index.ts index a110725..fd7f7c5 100644 --- a/src/vnext/statement-index.ts +++ b/src/vnext/statement-index.ts @@ -20,6 +20,12 @@ export { export type { SqlLexicalProfile } from "./lexical.js"; const analysisRangeBrand: unique symbol = Symbol("SqlAnalysisRange"); +const exactSqlStatementSlots = new WeakSet(); +const sqlStatementIndexProvenance = new WeakMap< + object, + SqlStatementProvenance +>(); +const sqlStatementSlots = new WeakSet(); export const MAX_SQL_STATEMENT_SLOTS = 10_000; const MAX_PREFIX_TOKENS = 6; @@ -87,6 +93,82 @@ export interface SqlStatementIndex { readonly slots: readonly SqlStatementSlot[]; } +interface SqlStatementProvenance { + readonly analysisText: string; + readonly profile: SqlLexicalProfile; + readonly slots: WeakSet; +} + +function snapshotLexicalProfile( + profile: SqlLexicalProfile, +): SqlLexicalProfile { + return Object.freeze({ + backtickQuotedIdentifiers: profile.backtickQuotedIdentifiers, + bigQueryStrings: profile.bigQueryStrings, + dollarQuotedStrings: profile.dollarQuotedStrings, + hashLineComments: profile.hashLineComments, + nestedBlockComments: profile.nestedBlockComments, + proceduralGuards: profile.proceduralGuards, + singleQuoteBackslash: profile.singleQuoteBackslash, + }); +} + +function hasSameLexicalProfile( + left: SqlLexicalProfile, + right: SqlLexicalProfile, +): boolean { + return ( + left.backtickQuotedIdentifiers === right.backtickQuotedIdentifiers && + left.bigQueryStrings === right.bigQueryStrings && + left.dollarQuotedStrings === right.dollarQuotedStrings && + left.hashLineComments === right.hashLineComments && + left.nestedBlockComments === right.nestedBlockComments && + left.proceduralGuards === right.proceduralGuards && + left.singleQuoteBackslash === right.singleQuoteBackslash + ); +} + +export function isExactSqlStatementSlotSnapshot( + candidate: unknown, +): candidate is ExactSqlStatementSlot { + return ( + candidate !== null && + typeof candidate === "object" && + exactSqlStatementSlots.has(candidate) + ); +} + +export function isExactSqlStatementSlotSnapshotFor( + index: unknown, + candidate: unknown, + analysisText: string, + profile: SqlLexicalProfile, +): candidate is ExactSqlStatementSlot { + if ( + index === null || + typeof index !== "object" || + !isExactSqlStatementSlotSnapshot(candidate) + ) { + return false; + } + const provenance = sqlStatementIndexProvenance.get(index); + return ( + provenance?.analysisText === analysisText && + hasSameLexicalProfile(provenance.profile, profile) && + provenance.slots.has(candidate) + ); +} + +export function isSqlStatementSlotSnapshot( + candidate: unknown, +): candidate is SqlStatementSlot { + return ( + candidate !== null && + typeof candidate === "object" && + sqlStatementSlots.has(candidate) + ); +} + const NORMAL_END_STATE: Extract< SqlLexicalEndState, { readonly kind: "normal" } @@ -122,7 +204,7 @@ function createExactSlot( hasCode: boolean, endState: ExactSqlStatementSlot["endState"], ): ExactSqlStatementSlot { - return Object.freeze({ + const slot = Object.freeze({ boundaryQuality: "exact", endState, extent: createAnalysisRange(from, extentTo), @@ -131,6 +213,9 @@ function createExactSlot( terminator: sourceTo === extentTo ? null : createAnalysisRange(sourceTo, extentTo), }); + exactSqlStatementSlots.add(slot); + sqlStatementSlots.add(slot); + return slot; } function createOpaqueSlot( @@ -139,11 +224,13 @@ function createOpaqueSlot( reason: SqlOpaqueBoundaryReason, detectedAt: number, ): OpaqueSqlStatementSlot { - return Object.freeze({ + const slot = Object.freeze({ boundaryQuality: "opaque", endState: createOpaqueEndState(reason, detectedAt), extent: createAnalysisRange(from, to), }); + sqlStatementSlots.add(slot); + return slot; } class SqlPrefixGuard { @@ -360,14 +447,27 @@ interface SqlStatementScanOptions { function createStatementIndex( slots: SqlStatementSlot[], + analysisText: string, + profile: SqlLexicalProfile, ): SqlStatementIndex { const finalSlot = getStatementSlot(slots, slots.length - 1); - return Object.freeze({ + const membership = new WeakSet(); + for (const slot of slots) { + membership.add(slot); + } + const provenance = Object.freeze({ + analysisText, + profile, + slots: membership, + }); + const index = Object.freeze({ endState: finalSlot.endState, quality: finalSlot.boundaryQuality === "opaque" ? "opaque" : "exact", slots: Object.freeze(slots), }); + sqlStatementIndexProvenance.set(index, provenance); + return index; } function scanSqlStatementIndex( @@ -393,7 +493,7 @@ function scanSqlStatementIndex( detectedAt, ); slots.push(slot); - return createStatementIndex(slots); + return createStatementIndex(slots, analysisText, profile); }; while (cursor < analysisText.length) { @@ -611,7 +711,7 @@ function scanSqlStatementIndex( finalEndState, ), ); - return createStatementIndex(slots); + return createStatementIndex(slots, analysisText, profile); } /** Builds the bounded, parser-free statement partition for one analysis text. */ @@ -619,10 +719,14 @@ export function buildSqlStatementIndex( analysisText: string, profile: SqlLexicalProfile, ): SqlStatementIndex { - return scanSqlStatementIndex(analysisText, profile, { - from: 0, - prefixSlots: [], - }); + return scanSqlStatementIndex( + analysisText, + snapshotLexicalProfile(profile), + { + from: 0, + prefixSlots: [], + }, + ); } function statementSlotIndexAt( @@ -697,7 +801,7 @@ function shiftStatementSlot( delta: number, ): SqlStatementSlot { if (slot.boundaryQuality === "opaque") { - return Object.freeze({ + const shifted = Object.freeze({ boundaryQuality: "opaque", endState: shiftEndState(slot.endState, delta), extent: createAnalysisRange( @@ -705,8 +809,10 @@ function shiftStatementSlot( slot.extent.to + delta, ), }); + sqlStatementSlots.add(shifted); + return shifted; } - return Object.freeze({ + const shifted = Object.freeze({ boundaryQuality: "exact", endState: shiftEndState(slot.endState, delta), extent: createAnalysisRange( @@ -725,6 +831,9 @@ function shiftStatementSlot( ) : null, }); + exactSqlStatementSlots.add(shifted); + sqlStatementSlots.add(shifted); + return shifted; } function normalizeTrustedChanges( @@ -774,15 +883,37 @@ export function updateSqlStatementIndex( changes: readonly SqlTextChange[], profile: SqlLexicalProfile, ): SqlStatementIndex { + const previousProvenance = + sqlStatementIndexProvenance.get(previousIndex); + if ( + !previousProvenance || + !hasSameLexicalProfile(previousProvenance.profile, profile) + ) { + return scanSqlStatementIndex( + nextAnalysisText, + snapshotLexicalProfile(profile), + { + from: 0, + prefixSlots: [], + }, + ); + } + const nextProfile = previousProvenance.profile; const previousSlots = previousIndex.slots; const previousLength = getStatementSlot( previousSlots, previousSlots.length - 1, ).extent.to; if (changes.length === 0) { - return previousLength === nextAnalysisText.length + const unchanged = + previousLength === nextAnalysisText.length && + previousProvenance.analysisText === nextAnalysisText; + return unchanged ? previousIndex - : buildSqlStatementIndex(nextAnalysisText, profile); + : scanSqlStatementIndex(nextAnalysisText, nextProfile, { + from: 0, + prefixSlots: [], + }); } const normalized = normalizeTrustedChanges( changes, @@ -790,7 +921,10 @@ export function updateSqlStatementIndex( nextAnalysisText.length, ); if (!normalized) { - return buildSqlStatementIndex(nextAnalysisText, profile); + return scanSqlStatementIndex(nextAnalysisText, nextProfile, { + from: 0, + prefixSlots: [], + }); } const restartIndex = statementSlotIndexAt( @@ -816,7 +950,7 @@ export function updateSqlStatementIndex( previousFinalSlot.boundaryQuality === "opaque" && previousFinalSlot.endState.reason === "resource-limit"; - return scanSqlStatementIndex(nextAnalysisText, profile, { + return scanSqlStatementIndex(nextAnalysisText, nextProfile, { from: restartFrom, prefixSlots, tryReuseSuffix: (newBoundary, scannedSlots) => { @@ -862,7 +996,11 @@ export function updateSqlStatementIndex( : oldSuffix.map((slot) => shiftStatementSlot(slot, normalized.delta), ); - return createStatementIndex([...scannedSlots, ...suffix]); + return createStatementIndex( + [...scannedSlots, ...suffix], + nextAnalysisText, + nextProfile, + ); }, }); }