From 89c191de564738cd672b76cdb62b023db02da649 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:09:50 +0800 Subject: [PATCH 01/25] feat(vnext): add statement gutter --- docs/vnext/codemirror-adapter.md | 34 ++ .../codemirror/__tests__/sql-editor.test.ts | 396 ++++++++++++++++-- .../browser_tests/statement-gutter.test.ts | 122 ++++++ src/vnext/codemirror/index.ts | 3 + src/vnext/codemirror/sql-editor.ts | 72 ++++ src/vnext/codemirror/statement-gutter.ts | 238 +++++++++++ ...o-relation-completion-codemirror.test-d.ts | 11 + 7 files changed, 852 insertions(+), 24 deletions(-) create mode 100644 src/vnext/codemirror/browser_tests/statement-gutter.test.ts create mode 100644 src/vnext/codemirror/statement-gutter.ts diff --git a/docs/vnext/codemirror-adapter.md b/docs/vnext/codemirror-adapter.md index 572d52a..8ab069d 100644 --- a/docs/vnext/codemirror-adapter.md +++ b/docs/vnext/codemirror-adapter.md @@ -70,6 +70,40 @@ const support = sqlEditor({ }); ``` +## Statement boundaries and gutter + +The support object exposes synchronous structural queries without exposing the +adapter-owned session: + +```ts +const current = support.statementBoundaryAt(view, { + affinity: "left", + position: view.state.selection.main.head, +}); +const visible = support.statementBoundariesIntersecting(view, { + from: view.viewport.from, + to: view.viewport.to, +}); +``` + +Both methods return `null` for foreign, destroyed, or unsynchronized views. +An opt-in structural gutter marks only lines intersecting scanner-owned SQL +`code` spans. It never parses or copies statement text: + +```ts +const support = sqlEditor({ + initialContext, + service, + statementGutter: { + hideWhenNotFocused: true, + showInactive: true, + }, +}); +``` + +The gutter uses `--cm-sql-statement-color` and +`--cm-sql-statement-inactive-opacity` CSS variables. It is disabled by default. + ## Atomic inputs Context and document changes can share one CodeMirror transaction: diff --git a/src/vnext/codemirror/__tests__/sql-editor.test.ts b/src/vnext/codemirror/__tests__/sql-editor.test.ts index 39ebef4..69e41c7 100644 --- a/src/vnext/codemirror/__tests__/sql-editor.test.ts +++ b/src/vnext/codemirror/__tests__/sql-editor.test.ts @@ -11,6 +11,7 @@ import { EditorSelection, type Extension } from "@codemirror/state"; import { EditorView } from "@codemirror/view"; import { afterEach, describe, expect, it, vi } from "vitest"; import { + bigQueryDialect, createSqlLanguageService, duckdbDialect, type SqlCatalogSearchRequest, @@ -27,6 +28,7 @@ import { type SqlRelationCatalogProvider, type SqlRevision, type SqlSessionChangeEvent, + type SqlTextRange, } from "../../index.js"; import { createSqlCompletionRefreshToken, @@ -48,6 +50,8 @@ interface FakeServiceHarness { readonly getLastToken: () => SqlCompletionRefreshToken | null; readonly service: SqlLanguageService; readonly sessionDisposals: () => number; + readonly statementBoundaryCalls: () => number; + readonly statementIntersectionCalls: () => number; readonly updates: readonly SqlDocumentUpdate[]; } @@ -126,6 +130,7 @@ function fakeService( token: SqlCompletionRefreshToken, ) => SqlCompletionResult | Promise, rejectUpdates = false, + statementCode: SqlTextRange | null = null, ): FakeServiceHarness { const completeSignals: AbortSignal[] = []; const updates: SqlDocumentUpdate[] = []; @@ -134,6 +139,8 @@ function fakeService( | null = null; let lastToken: SqlCompletionRefreshToken | null = null; let sessionDisposalCount = 0; + let statementBoundaryCallCount = 0; + let statementIntersectionCallCount = 0; const service: SqlLanguageService = { dispose: () => undefined, openDocument: () => { @@ -166,30 +173,56 @@ function fakeService( get revision() { return revision; }, - statementBoundaryAt: () => ({ - boundary: { - boundaryQuality: "exact", - code: null, - endState: { kind: "normal" }, - extent: { from: 0, to: 0 }, - hasCode: false, - source: { from: 0, to: 0 }, - terminator: null, - }, - revision, - }), - statementBoundariesIntersecting: () => ({ - boundaries: [{ - boundaryQuality: "exact", - code: null, - endState: { kind: "normal" }, - extent: { from: 0, to: 0 }, - hasCode: false, - source: { from: 0, to: 0 }, - terminator: null, - }], - revision, - }), + statementBoundaryAt: () => { + statementBoundaryCallCount += 1; + return { + boundary: statementCode === null + ? { + boundaryQuality: "exact", + code: null, + endState: { kind: "normal" }, + extent: { from: 0, to: 0 }, + hasCode: false, + source: { from: 0, to: 0 }, + terminator: null, + } + : { + boundaryQuality: "exact", + code: statementCode, + endState: { kind: "normal" }, + extent: statementCode, + hasCode: true, + source: statementCode, + terminator: null, + }, + revision, + }; + }, + statementBoundariesIntersecting: () => { + statementIntersectionCallCount += 1; + return { + boundaries: [statementCode === null + ? { + boundaryQuality: "exact", + code: null, + endState: { kind: "normal" }, + extent: { from: 0, to: 0 }, + hasCode: false, + source: { from: 0, to: 0 }, + terminator: null, + } + : { + boundaryQuality: "exact", + code: statementCode, + endState: { kind: "normal" }, + extent: statementCode, + hasCode: true, + source: statementCode, + terminator: null, + }], + revision, + }; + }, update: (update) => { updates.push(update); if (rejectUpdates) { @@ -208,6 +241,9 @@ function fakeService( getLastToken: () => lastToken, service, sessionDisposals: () => sessionDisposalCount, + statementBoundaryCalls: () => statementBoundaryCallCount, + statementIntersectionCalls: () => + statementIntersectionCallCount, updates, }; } @@ -320,6 +356,318 @@ async function resolveCompletionInfo( } describe("sqlEditor", () => { + it("exposes current statement boundaries only for owned views", () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + }); + const view = createView(support.extension, "SELECT 1;SELECT 2"); + const foreign = createView([]); + + expect(support.statementBoundaryAt(view, { + affinity: "left", + position: 9, + })?.boundary).toMatchObject({ + boundaryQuality: "exact", + code: { from: 0, to: 8 }, + hasCode: true, + }); + expect(support.statementBoundariesIntersecting(view, { + from: 0, + to: view.state.doc.length, + })?.boundaries).toHaveLength(2); + expect(support.statementBoundaryAt(foreign, { + affinity: "left", + position: 0, + })).toBeNull(); + expect(support.statementBoundariesIntersecting(foreign, { + from: 0, + to: 0, + })).toBeNull(); + expect(support.statementBoundaryAt(view, { + affinity: "left", + position: 100, + })).toBeNull(); + + view.destroy(); + expect(support.statementBoundaryAt(view, { + affinity: "left", + position: 0, + })).toBeNull(); + expect(support.statementBoundariesIntersecting(view, { + from: 0, + to: 0, + })).toBeNull(); + service.dispose(); + }); + + it("renders an opt-in visible-line statement gutter", async () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: {}, + }); + const view = createView( + support.extension, + "SELECT 1;\n\n/* separator */\nSELECT 2;", + ); + + await vi.waitFor(() => { + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(2); + }); + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker-active"), + ).toHaveLength(1); + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker-inactive"), + ).toHaveLength(1); + expect(Array.from( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).findIndex((marker) => + marker.classList.contains("cm-sql-statement-marker-active") + )).toBe(1); + + view.dispatch({ selection: { anchor: 1 } }); + await vi.waitFor(() => { + const markers = view.dom.querySelectorAll( + ".cm-sql-statement-marker", + ); + expect(markers).toHaveLength(2); + expect( + view.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ), + ).toHaveLength(1); + expect(Array.from(markers).findIndex((marker) => + marker.classList.contains( + "cm-sql-statement-marker-active", + ) + )).toBe(0); + }); + service.dispose(); + }); + + it("does not install a statement gutter by default", () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + }); + const view = createView(support.extension, "SELECT 1"); + + expect( + view.dom.querySelector(".cm-sql-statement-gutter"), + ).toBeNull(); + service.dispose(); + }); + + it("renders no marker for an empty document", () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: {}, + }); + const view = createView(support.extension, ""); + + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(0); + service.dispose(); + }); + + it("supports hidden and active-only gutter policies", async () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const hidden = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: { hideWhenNotFocused: true }, + }); + const hiddenView = createView(hidden.extension, "SELECT 1"); + expect( + hiddenView.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(0); + hiddenView.focus(); + await vi.waitFor(() => { + expect( + hiddenView.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(1); + }); + + const activeOnly = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: { showInactive: false }, + }); + const activeView = createView( + activeOnly.extension, + "SELECT 1;\nSELECT 2", + ); + await vi.waitFor(() => { + expect( + activeView.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ), + ).toHaveLength(1); + expect( + activeView.dom.querySelectorAll( + ".cm-sql-statement-marker-inactive", + ), + ).toHaveLength(0); + }); + activeOnly.setContext(activeView, { + dialect: "duckdb", + engine: "remote", + }); + await vi.waitFor(() => { + expect( + activeView.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ), + ).toHaveLength(1); + }); + service.dispose(); + }); + + it("shows only inactive markers when no code boundary is current", async () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: {}, + }); + const view = createView( + support.extension, + "SELECT 1;\n/* trailing */", + ); + + await vi.waitFor(() => { + expect( + view.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ), + ).toHaveLength(0); + expect( + view.dom.querySelectorAll( + ".cm-sql-statement-marker-inactive", + ), + ).toHaveLength(1); + }); + service.dispose(); + expect(() => { + view.dispatch({ selection: { anchor: 0 } }); + }).not.toThrow(); + }); + + it("accepts an explicit false gutter option", () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: false, + }); + const view = createView(support.extension, "SELECT 1"); + expect( + view.dom.querySelector(".cm-sql-statement-gutter"), + ).toBeNull(); + service.dispose(); + }); + + it("marks internal blank lines but not separator trivia", async () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: {}, + }); + const view = createView( + support.extension, + "SELECT\n\n 1;\n\nSELECT 2", + ); + + await vi.waitFor(() => { + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(4); + }); + service.dispose(); + }); + + it("does not fall back across an opaque right boundary", async () => { + const service = createSqlLanguageService({ + dialects: [bigQueryDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "bigquery", engine: "local" }, + service, + statementGutter: {}, + }); + const documentText = + "SELECT 1; IF condition THEN SELECT 2; END IF;"; + const sharedBoundary = documentText.indexOf(";") + 1; + const view = createView(support.extension, documentText); + view.dispatch({ selection: { anchor: sharedBoundary } }); + + expect( + support.statementBoundaryAt(view, { + affinity: "right", + position: sharedBoundary, + })?.boundary.boundaryQuality, + ).toBe("opaque"); + await vi.waitFor(() => { + expect( + view.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ), + ).toHaveLength(0); + }); + service.dispose(); + }); + + it("queries structural boundaries once per relevant redraw", () => { + const documentText = "SELECT\n\n 1;\nSELECT 2"; + const harness = fakeService( + (revision) => readyResult(revision, []), + false, + { from: 0, to: documentText.length }, + ); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service: harness.service, + statementGutter: {}, + }); + const view = createView(support.extension, documentText); + + expect(harness.statementBoundaryCalls()).toBe(1); + expect(harness.statementIntersectionCalls()).toBe(1); + view.dispatch({ selection: { anchor: 1 } }); + expect(harness.statementBoundaryCalls()).toBe(2); + expect(harness.statementIntersectionCalls()).toBe(2); + view.dispatch({}); + expect(harness.statementBoundaryCalls()).toBe(2); + expect(harness.statementIntersectionCalls()).toBe(2); + }); + it("maps current service completions and applies the exact core edit", async () => { const service = createSqlLanguageService({ catalog: { diff --git a/src/vnext/codemirror/browser_tests/statement-gutter.test.ts b/src/vnext/codemirror/browser_tests/statement-gutter.test.ts new file mode 100644 index 0000000..147d763 --- /dev/null +++ b/src/vnext/codemirror/browser_tests/statement-gutter.test.ts @@ -0,0 +1,122 @@ +import { EditorView } from "@codemirror/view"; +import { expect, test } from "vitest"; +import { + createSqlLanguageService, + duckdbDialect, +} from "../../index.js"; +import { sqlEditor } from "../index.js"; + +test("vNext statement gutter follows the current statement", async () => { + const parent = document.createElement("div"); + parent.style.height = "240px"; + document.body.append(parent); + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb" }, + service, + statementGutter: {}, + }); + const documentText = "SELECT 1;\n\n/* separator */\nSELECT 2;"; + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + selection: { anchor: documentText.length }, + }); + view.focus(); + + await expect.poll(() => + view.dom.querySelectorAll(".cm-sql-statement-marker").length + ).toBe(2); + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker-active"), + ).toHaveLength(1); + expect(Array.from( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).findIndex((marker) => + marker.classList.contains("cm-sql-statement-marker-active") + )).toBe(1); + + view.dispatch({ selection: { anchor: 1 } }); + await expect.poll(() => { + const markers = Array.from( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ); + return markers.findIndex((marker) => + marker.classList.contains("cm-sql-statement-marker-active") + ); + }).toBe(0); + + view.destroy(); + service.dispose(); + parent.remove(); +}); + +test("vNext statement gutter virtualizes a tall focused editor", async () => { + const parent = document.createElement("div"); + parent.style.height = "120px"; + parent.style.setProperty( + "--cm-sql-statement-color", + "rgb(1, 2, 3)", + ); + document.body.append(parent); + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb" }, + service, + statementGutter: { hideWhenNotFocused: true }, + }); + const documentText = Array.from( + { length: 200 }, + (_, index) => `SELECT\n\n ${index};`, + ).join("\n/* separator */\n"); + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + }); + + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(0); + view.focus(); + await expect.poll(() => + view.dom.querySelectorAll(".cm-sql-statement-marker").length + ).toBeGreaterThan(0); + const initialCount = view.dom.querySelectorAll( + ".cm-sql-statement-marker", + ).length; + expect(initialCount).toBeLessThan(200); + const marker = view.dom.querySelector( + ".cm-sql-statement-marker", + ); + expect(marker).not.toBeNull(); + expect(getComputedStyle(marker!).width).toBe("3px"); + expect(getComputedStyle(marker!).backgroundColor).toBe( + "rgb(1, 2, 3)", + ); + + view.dispatch({ + effects: EditorView.scrollIntoView(documentText.length, { + y: "start", + }), + selection: { anchor: documentText.length }, + }); + await expect.poll(() => view.viewport.from).toBeGreaterThan(0); + await expect.poll(() => + view.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ).length + ).toBe(3); + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker").length, + ).toBeLessThan(200); + + view.destroy(); + service.dispose(); + parent.remove(); +}); diff --git a/src/vnext/codemirror/index.ts b/src/vnext/codemirror/index.ts index e5e1d6f..16e2265 100644 --- a/src/vnext/codemirror/index.ts +++ b/src/vnext/codemirror/index.ts @@ -4,6 +4,9 @@ export { type SqlEditorOptions, type SqlEditorSupport, } from "./sql-editor.js"; +export type { + SqlEditorStatementGutterOptions, +} from "./statement-gutter.js"; export type { SqlCompletionInfoResolver, SqlCompletionInfoResolverContext, diff --git a/src/vnext/codemirror/sql-editor.ts b/src/vnext/codemirror/sql-editor.ts index 652b143..54a0cd6 100644 --- a/src/vnext/codemirror/sql-editor.ts +++ b/src/vnext/codemirror/sql-editor.ts @@ -36,6 +36,12 @@ import type { SqlCompletionResult, SqlCompletionTask, } from "../relation-completion-types.js"; +import type { + SqlStatementBoundariesIntersectingRequest, + SqlStatementBoundariesIntersectingResult, + SqlStatementBoundaryAtRequest, + SqlStatementBoundaryAtResult, +} from "../statement-boundary-types.js"; import type { SqlContextInput, SqlDocumentContext, @@ -45,6 +51,10 @@ import type { SqlRevision, SqlTextChange, } from "../types.js"; +import { + createSqlStatementGutter, + type SqlEditorStatementGutterOptions, +} from "./statement-gutter.js"; export interface SqlEditorAutocompleteOptions { readonly activateOnTyping?: boolean; @@ -67,6 +77,9 @@ export interface SqlEditorOptions< | readonly SqlEmbeddedRegion[] | undefined; readonly service: SqlLanguageService; + readonly statementGutter?: + | false + | SqlEditorStatementGutterOptions; } export interface SqlEditorSupport< @@ -77,6 +90,14 @@ export interface SqlEditorSupport< readonly SqlEmbeddedRegion[] >; readonly extension: Extension; + readonly statementBoundariesIntersecting: ( + view: EditorView, + request: SqlStatementBoundariesIntersectingRequest, + ) => SqlStatementBoundariesIntersectingResult | null; + readonly statementBoundaryAt: ( + view: EditorView, + request: SqlStatementBoundaryAtRequest, + ) => SqlStatementBoundaryAtResult | null; readonly setContext: ( view: EditorView, context: SqlContextInput, @@ -607,6 +628,28 @@ export function createSqlEditorInternal< this.#clearCompletionState(); }; + readonly statementBoundariesIntersecting = ( + request: SqlStatementBoundariesIntersectingRequest, + ): SqlStatementBoundariesIntersectingResult | null => { + if (this.#destroyed) return null; + try { + return this.#session.statementBoundariesIntersecting(request); + } catch { + return null; + } + }; + + readonly statementBoundaryAt = ( + request: SqlStatementBoundaryAtRequest, + ): SqlStatementBoundaryAtResult | null => { + if (this.#destroyed) return null; + try { + return this.#session.statementBoundaryAt(request); + } catch { + return null; + } + }; + readonly update = (update: ViewUpdate): void => { let contextChanged = false; let regionsChanged = false; @@ -744,6 +787,23 @@ export function createSqlEditorInternal< }, }]), ); + const statementGutter = options.statementGutter === false || + options.statementGutter === undefined + ? [] + : createSqlStatementGutter(options.statementGutter, { + boundariesIntersecting: (view, range) => + view.plugin(plugin)?.statementBoundariesIntersecting(range) ?? + null, + boundaryAt: (view, position, affinity) => + view.plugin(plugin)?.statementBoundaryAt({ + affinity, + position, + }) ?? null, + inputKeys: (view) => [ + view.state.field(contextField), + view.state.field(embeddedRegionsField), + ], + }); return Object.freeze({ contextEffect, embeddedRegionsEffect, @@ -752,11 +812,23 @@ export function createSqlEditorInternal< embeddedRegionsField, plugin, escapeKeymap, + statementGutter, autocompletion({ ...autocompleteOptions, override: [completionSource, ...externalSources], }), ], + statementBoundariesIntersecting: ( + view: EditorView, + request: SqlStatementBoundariesIntersectingRequest, + ) => + view.plugin(plugin)?.statementBoundariesIntersecting(request) ?? + null, + statementBoundaryAt: ( + view: EditorView, + request: SqlStatementBoundaryAtRequest, + ) => + view.plugin(plugin)?.statementBoundaryAt(request) ?? null, setContext: ( view: EditorView, context: SqlContextInput, diff --git a/src/vnext/codemirror/statement-gutter.ts b/src/vnext/codemirror/statement-gutter.ts new file mode 100644 index 0000000..392adee --- /dev/null +++ b/src/vnext/codemirror/statement-gutter.ts @@ -0,0 +1,238 @@ +import { + RangeSet, + type Extension, + type Text, +} from "@codemirror/state"; +import { + EditorView, + GutterMarker, + gutter, +} from "@codemirror/view"; +import type { + SqlStatementBoundariesIntersectingResult, + SqlStatementBoundaryAtResult, + SqlTextRange, +} from "../index.js"; + +export interface SqlEditorStatementGutterOptions { + readonly hideWhenNotFocused?: boolean; + readonly showInactive?: boolean; +} + +export interface SqlStatementGutterAccess { + readonly boundaryAt: ( + view: EditorView, + position: number, + affinity: "left" | "right", + ) => SqlStatementBoundaryAtResult | null; + readonly boundariesIntersecting: ( + view: EditorView, + range: SqlTextRange, + ) => SqlStatementBoundariesIntersectingResult | null; + readonly inputKeys: ( + view: EditorView, + ) => readonly [unknown, unknown]; +} + +class SqlStatementGutterMarker extends GutterMarker { + constructor(readonly active: boolean) { + super(); + } + + override eq(other: SqlStatementGutterMarker): boolean { + return this.active === other.active; + } + + override toDOM(view: EditorView): Node { + const marker = view.dom.ownerDocument.createElement("div"); + marker.className = this.active + ? "cm-sql-statement-marker cm-sql-statement-marker-active" + : "cm-sql-statement-marker cm-sql-statement-marker-inactive"; + return marker; + } +} + +const activeMarker = new SqlStatementGutterMarker(true); +const inactiveMarker = new SqlStatementGutterMarker(false); +const emptyMarkers: RangeSet = RangeSet.empty; + +interface SqlStatementGutterSnapshot { + readonly contextKey: unknown; + readonly document: Text; + readonly embeddedRegionsKey: unknown; + readonly focused: boolean; + readonly markers: RangeSet; + readonly selectionHead: number; + readonly viewportFrom: number; + readonly viewportTo: number; +} + +function sameRange( + left: SqlTextRange, + right: SqlTextRange, +): boolean { + return left.from === right.from && left.to === right.to; +} + +function exactCode( + result: SqlStatementBoundaryAtResult | null, +): SqlTextRange | null { + const boundary = result?.boundary; + return boundary?.boundaryQuality === "exact" && + boundary.hasCode + ? boundary.code + : null; +} + +function currentCodeRange( + view: EditorView, + access: SqlStatementGutterAccess, +): SqlTextRange | null { + const position = view.state.selection.main.head; + const right = access.boundaryAt(view, position, "right"); + if ( + right?.boundary.boundaryQuality === "opaque" + ) { + return null; + } + const rightCode = exactCode(right); + if (rightCode !== null) return rightCode; + if ( + right?.boundary.boundaryQuality !== "exact" || + right.boundary.hasCode + ) { + return null; + } + return exactCode( + access.boundaryAt(view, position, "left"), + ); +} + +function buildMarkers( + view: EditorView, + options: SqlEditorStatementGutterOptions, + access: SqlStatementGutterAccess, +): RangeSet { + if ( + options.hideWhenNotFocused === true && !view.hasFocus + ) { + return emptyMarkers; + } + const { from, to } = view.viewport; + if (from === to) return emptyMarkers; + const visible = access.boundariesIntersecting(view, { from, to }); + if (visible === null) return emptyMarkers; + const current = currentCodeRange(view, access); + const lineMarkers = new Map(); + for (const boundary of visible.boundaries) { + if ( + boundary.boundaryQuality !== "exact" || + !boundary.hasCode + ) { + continue; + } + const codeFrom = Math.max(from, boundary.code.from); + const codeTo = Math.min(to, boundary.code.to); + if (codeFrom >= codeTo) continue; + const active = current !== null && + sameRange(boundary.code, current); + let line = view.state.doc.lineAt(codeFrom); + while (line.from < codeTo) { + if (active || options.showInactive !== false) { + lineMarkers.set( + line.from, + active || lineMarkers.get(line.from) === true, + ); + } + if (line.to >= codeTo || line.to === view.state.doc.length) { + break; + } + line = view.state.doc.line(line.number + 1); + } + } + return RangeSet.of( + Array.from(lineMarkers, ([position, active]) => + (active ? activeMarker : inactiveMarker).range(position) + ), + true, + ); +} + +function snapshotMatches( + snapshot: SqlStatementGutterSnapshot, + view: EditorView, + contextKey: unknown, + embeddedRegionsKey: unknown, +): boolean { + return snapshot.document === view.state.doc && + snapshot.selectionHead === view.state.selection.main.head && + snapshot.viewportFrom === view.viewport.from && + snapshot.viewportTo === view.viewport.to && + snapshot.focused === view.hasFocus && + snapshot.contextKey === contextKey && + snapshot.embeddedRegionsKey === embeddedRegionsKey; +} + +export function createSqlStatementGutter( + options: SqlEditorStatementGutterOptions, + access: SqlStatementGutterAccess, +): Extension { + const snapshots = new WeakMap< + EditorView, + SqlStatementGutterSnapshot + >(); + return [ + gutter({ + class: "cm-sql-statement-gutter", + markers: (view) => { + const [contextKey, embeddedRegionsKey] = + access.inputKeys(view); + const previous = snapshots.get(view); + if ( + previous !== undefined && + snapshotMatches( + previous, + view, + contextKey, + embeddedRegionsKey, + ) + ) { + return previous.markers; + } + const markers = buildMarkers(view, options, access); + snapshots.set(view, { + contextKey, + document: view.state.doc, + embeddedRegionsKey, + focused: view.hasFocus, + markers, + selectionHead: view.state.selection.main.head, + viewportFrom: view.viewport.from, + viewportTo: view.viewport.to, + }); + return markers; + }, + }), + EditorView.baseTheme({ + ".cm-sql-statement-gutter": { + minWidth: "3px", + width: "3px", + }, + ".cm-sql-statement-gutter .cm-gutterElement": { + margin: "0", + padding: "0", + width: "3px", + }, + ".cm-sql-statement-marker": { + backgroundColor: "var(--cm-sql-statement-color, #3b82f6)", + borderRadius: "1px", + display: "block", + height: "100%", + width: "3px", + }, + ".cm-sql-statement-marker-inactive": { + opacity: "var(--cm-sql-statement-inactive-opacity, 0.3)", + }, + }), + ]; +} diff --git a/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts b/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts index 33938c2..95dab26 100644 --- a/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts +++ b/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts @@ -3,6 +3,7 @@ import type { EditorView } from "@codemirror/view"; import type { SqlCompletionInfoResolver, SqlCompletionInfoResolverContext, + SqlEditorStatementGutterOptions, } from "../../src/vnext/codemirror/index.js"; import type { SqlCompletionItem } from "../../src/vnext/relation-completion-types.js"; @@ -32,6 +33,14 @@ declare const item: SqlCompletionItem; const resolved = resolveInfo(item, { signal: new AbortController().signal, }); +const gutter: SqlEditorStatementGutterOptions = { + hideWhenNotFocused: true, + showInactive: true, +}; +const invalidGutter: SqlEditorStatementGutterOptions = { + // @ts-expect-error gutter flags are boolean + showInactive: "yes", +}; // @ts-expect-error resolver items are immutable item.label = "changed"; @@ -62,6 +71,8 @@ const resolverWithoutDestroy: SqlCompletionInfoResolver = () => ({ }); void resolved; +void gutter; +void invalidGutter; void resolverReturningNode; void resolverReturningNumber; void resolverReturningReactData; From 4a3de62f9cbc1483384df034f55c85315927fd90 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:29:16 +0800 Subject: [PATCH 02/25] docs(vnext): model marimo SQL migration --- docs/vnext/marimo-sql-migration.md | 152 ++++++++ .../marimo-sql-migration.test-d.ts | 363 ++++++++++++++++++ 2 files changed, 515 insertions(+) create mode 100644 docs/vnext/marimo-sql-migration.md create mode 100644 test/vnext-types/marimo-sql-migration.test-d.ts diff --git a/docs/vnext/marimo-sql-migration.md b/docs/vnext/marimo-sql-migration.md new file mode 100644 index 0000000..7cb7285 --- /dev/null +++ b/docs/vnext/marimo-sql-migration.md @@ -0,0 +1,152 @@ +# Marimo SQL Completion Migration + +Status: implementation fixture; relation completion only + +The compile-only fixture +[`marimo-sql-migration.test-d.ts`](../../test/vnext-types/marimo-sql-migration.test-d.ts) +models the intended marimo integration exclusively through public vNext +exports. It is not application code and has no runtime dependency on marimo. + +## Ownership and provider seam + +Marimo creates one caller-owned `SqlLanguageService` for an editor fleet. +Every `sqlEditor` support/view opens its own session against that service. +Destroying a view never disposes the service; the application disposes the +service after its views have been retired. + +One provider projects `dataConnectionsMapAtom` and `datasetTablesAtom` into +canonical relation records. It must preserve today's collision rule: +connection relations win over same-named local relations. Provider records +contain plain immutable data and stable entity IDs, never Jotai atoms, React +roots, or backend objects. + +Catalog `scope` identifies a live connection incarnation: + +```text +: +``` + +Replacing or reconnecting a same-named connection creates a new incarnation +and scope. Schema/table mutations within an incarnation advance its provider +epoch and notify the one scoped subscription. The context `searchPath` values +reproduce +`default_database`, `default_schema`, nested-schema, and schemaless behavior. + +## One atomic editor input + +Marimo should have one transaction builder for SQL interpretation changes. A +document or engine change emits all of these in the same CodeMirror +transaction: + +1. marimo language metadata; +2. `support.contextEffect` with engine, registered vNext dialect, connection + scope, and search paths; +3. the CodeMirror SQL dialect compartment reconfiguration; and +4. `support.embeddedRegionsEffect` containing the complete region set in the + resulting document. + +No update listener should dispatch a follow-up context or dialect transaction. +That creates an observable grammar/catalog mismatch and makes rapid +connection switches race. + +## Templates and external sources + +`sqlEditor` owns the single `autocompletion()` configuration. Marimo passes its +Python variable and keyword sources through `autocomplete.externalSources`. +The region scanner recognizes single `{python}` expressions, ignores escaped +`{{`, includes both delimiters in each region, and supplies a complete sorted +set after every document change. SQL completion is unavailable inside those +regions while the external variable source remains active. + +There is one unresolved insertion-point edge case. Embedded regions are +non-empty half-open ranges. An unmatched `{` ending at the document boundary +cannot contain the cursor position at `doc.length`, so regions alone cannot +suppress SQL completion at that exact point. Before cutover, either: + +- add an adapter completion gate for host-owned embedded insertion points; +- extend the core template contract with an explicit open-ended cursor + barrier; or +- keep a routed completion source that returns only variable results for the + unmatched-expression site. + +Do not encode a range beyond the document or an empty region. + +The rich-info resolver uses catalog provenance IDs to look up current marimo +metadata, mounts React into a new element, and returns +`{ dom, destroy: root.unmount }`. The adapter owns cancellation and disposal. + +## Dialect cutover + +The first cutover is limited to the registered vNext dialects: + +- DuckDB +- PostgreSQL +- BigQuery +- Dremio + +MySQL, MariaDB, SQLite, MSSQL, Oracle, and standard/fallback engines continue +to use the legacy schema completion source. The router must select exactly one +table/schema source; installing legacy schema completion as an external source +for a vNext-enabled dialect would create duplicate and conflicting relation +edits. + +Full removal of the legacy source requires equivalent dialect handles or an +explicit, tested generic-dialect policy. + +## Remaining library gaps + +Marimo's current `tablesCompletionSource()` is broader than its name. The +CodeMirror SQL schema source provides relations, namespace navigation, and +columns. vNext currently provides relation items only. + +No-regression replacement therefore still requires: + +- lazy, bounded, cancellation-aware column lookup keyed by stable relation + entity IDs; +- qualified and unqualified column completion with aliases, joins, + ambiguity, quoting, and exact edits; +- column provenance for the disposable info resolver; +- namespace/container parity for database, schema, project, and dataset + navigation; and +- the dialect coverage described above. + +Column lookup should be batched for all visible relations at a completion site. +It must not attach every column to every relation-search response or issue one +provider request per relation. + +## Migration sequence + +1. Capture golden legacy results for labels, kinds, edit ranges, qualification, + and details across representative connection shapes. +2. Land the shared provider, incarnation scope, atomic transaction builder, + region scanner, and completion router behind a feature flag. +3. Run vNext relation completion in shadow mode while the legacy source remains + visible. +4. Add column and namespace completion to the library and compare the golden + corpus. +5. Cut over the four supported dialects as one source replacement, preserving + variable and keyword external sources. +6. Add dialect coverage, expand the router, and remove completion-only legacy + schema code. Keep legacy schema data while hover or diagnostics still use + it. + +## Acceptance tests + +Library tests cover relation and column completion for `FROM`, `JOIN`, +`alias.`, unqualified projections and predicates, `USING`, nested queries, +CTEs, quoted identifiers, ambiguous columns, provider loading/invalidations, +bounded batching, and template barriers. + +Marimo integration tests cover: + +- default, nested, and schemaless database/schema layouts; +- local-table collisions where the connection relation wins; +- atomic engine/dialect/scope changes and rapid A-B-A switching; +- reconnecting under the same display name without stale cache reuse; +- open-menu schema invalidation; +- closed, escaped, and unmatched Python expressions; +- completion edits mapping regions in the same transaction; +- variable and keyword source preservation; +- React table/column info unmount on navigation, edit, and view destruction; +- one shared service across 1, 10, and 50 views; and +- exclusive legacy routing for unsupported dialects. diff --git a/test/vnext-types/marimo-sql-migration.test-d.ts b/test/vnext-types/marimo-sql-migration.test-d.ts new file mode 100644 index 0000000..ab2d76b --- /dev/null +++ b/test/vnext-types/marimo-sql-migration.test-d.ts @@ -0,0 +1,363 @@ +import type { + CompletionSource, +} from "@codemirror/autocomplete"; +import type { + ChangeSpec, + StateEffectType, + TransactionSpec, +} from "@codemirror/state"; + +import { + bigQueryDialect, + createSqlLanguageService, + dremioDialect, + duckdbDialect, + postgresDialect, + type SqlCanonicalRelationPath, + type SqlCatalogEpoch, + type SqlCatalogReadyCoverage, + type SqlCatalogRelation, + type SqlCatalogSearchRequest, + type SqlCatalogSearchResponse, + type SqlContextInput, + type SqlDocumentContext, + type SqlEmbeddedRegion, + type SqlIdentifierPath, + type SqlRelationCatalogProvider, +} from "../../src/vnext/index.js"; +import { + sqlEditor, + type SqlCompletionInfoResolver, + type SqlEditorSupport, +} from "../../src/vnext/codemirror/index.js"; + +type VnextDialectId = + | "bigquery" + | "dremio" + | "duckdb" + | "postgres"; + +type MarimoBackendDialect = + | VnextDialectId + | "postgresql" + | "mysql" + | "oracle" + | "sqlite" + | "snowflake"; + +interface MarimoSqlContext extends SqlDocumentContext { + readonly engine: string; +} + +interface MarimoConnection { + readonly defaultDatabase: string | null; + readonly defaultSchema: string | null; + readonly dialect: MarimoBackendDialect; + readonly engine: string; + /** Changes whenever a same-named backend connection is replaced. */ + readonly incarnation: number; +} + +interface MarimoTableMetadata { + readonly detail: string; + readonly entityId: string; +} + +interface MarimoCatalogSnapshot { + readonly epoch: SqlCatalogEpoch; + readonly relations: readonly SqlCatalogRelation[]; +} + +declare const catalogByScope: + ReadonlyMap; +declare const tableMetadataById: + ReadonlyMap; +declare const subscribeToCatalogScope: ( + scope: string, + listener: (epoch: SqlCatalogEpoch) => void, +) => () => void; +declare const searchMarimoCatalogIndex: ( + snapshot: MarimoCatalogSnapshot, + request: SqlCatalogSearchRequest, +) => { + readonly coverage: SqlCatalogReadyCoverage; + readonly relations: readonly SqlCatalogRelation[]; +}; +declare const variableCompletionSource: CompletionSource; +declare const keywordCompletionSource: CompletionSource; +declare const legacySchemaCompletionSource: CompletionSource; + +interface ReactRootLike { + readonly render: (value: unknown) => void; + readonly unmount: () => void; +} + +declare function createReactRoot(container: Element): ReactRootLike; + +function connectionScope( + connection: MarimoConnection, +): string { + return `${connection.engine}:${connection.incarnation}`; +} + +function identifier(value: string): SqlIdentifierPath[number] { + return { quoted: false, value }; +} + +function searchPaths( + connection: MarimoConnection, +): readonly SqlIdentifierPath[] { + const path: SqlIdentifierPath[number][] = []; + if (connection.defaultDatabase !== null) { + path.push(identifier(connection.defaultDatabase)); + } + if (connection.defaultSchema !== null) { + path.push(identifier(connection.defaultSchema)); + } + return path.length === 0 ? [] : [path]; +} + +function contextFor( + route: VnextCompletionRoute, +): SqlContextInput { + const { connection } = route; + return { + catalog: { + scope: connectionScope(connection), + searchPath: searchPaths(connection), + }, + dialect: route.dialect, + engine: connection.engine, + }; +} + +const marimoCatalogProvider: SqlRelationCatalogProvider = { + id: "marimo-datasets", + search: async ( + request, + signal, + ): Promise => { + signal.throwIfAborted(); + const snapshot = catalogByScope.get(request.scope); + if (snapshot === undefined) { + return { + code: "invalid-configuration", + epoch: { generation: 0, token: "missing-scope" }, + retry: "after-invalidation", + status: "failed", + }; + } + const result = searchMarimoCatalogIndex(snapshot, request); + return { + coverage: result.coverage, + epoch: snapshot.epoch, + relations: result.relations, + status: "ready", + }; + }, + subscribe: (scope, onInvalidation) => { + const unsubscribe = subscribeToCatalogScope(scope, (epoch) => { + onInvalidation({ epoch }); + }); + return (): undefined => { + unsubscribe(); + return undefined; + }; + }, +}; + +// One caller-owned service is shared by every SQL editor support/view. +const sharedSqlService = + createSqlLanguageService({ + catalog: marimoCatalogProvider, + dialects: [ + bigQueryDialect(), + dremioDialect(), + duckdbDialect(), + postgresDialect(), + ], + }); + +const infoResolver: SqlCompletionInfoResolver = ( + item, + { signal }, +) => { + signal.throwIfAborted(); + if (item.provenance.kind !== "catalog") return null; + const metadata = tableMetadataById.get( + item.provenance.entityId, + ); + if (metadata === undefined) return null; + const dom = document.createElement("div"); + const root = createReactRoot(dom); + root.render(metadata.detail); + return { + destroy: () => root.unmount(), + dom, + }; +}; + +/** + * Returns the complete, ordered region set for the resulting document. + * Regions include both braces, matching marimo's current `{python}` syntax. + */ +function pythonTemplateRegions( + text: string, +): readonly SqlEmbeddedRegion[] { + const regions: SqlEmbeddedRegion[] = []; + for (let index = 0; index < text.length;) { + if (text[index] !== "{") { + index += 1; + continue; + } + if (text[index + 1] === "{") { + index += 2; + continue; + } + const close = text.indexOf("}", index + 1); + const to = close === -1 ? text.length : close + 1; + if (to > index) { + regions.push({ + from: index, + language: "python", + to, + }); + } + index = Math.max(index + 1, to); + } + return regions; +} + +interface VnextCompletionRoute { + readonly connection: MarimoConnection; + readonly dialect: VnextDialectId; + readonly kind: "vnext"; +} + +type CompletionRoute = + | { + readonly kind: "legacy"; + readonly source: CompletionSource; + } + | VnextCompletionRoute; + +function completionRoute( + connection: MarimoConnection, +): CompletionRoute { + switch (connection.dialect) { + case "bigquery": + return { connection, dialect: "bigquery", kind: "vnext" }; + case "dremio": + return { connection, dialect: "dremio", kind: "vnext" }; + case "duckdb": + return { connection, dialect: "duckdb", kind: "vnext" }; + case "postgres": + case "postgresql": + return { connection, dialect: "postgres", kind: "vnext" }; + case "mysql": + case "oracle": + case "sqlite": + case "snowflake": + return { kind: "legacy", source: legacySchemaCompletionSource }; + } +} + +function createVnextSupport( + route: VnextCompletionRoute, + initialText: string, +): SqlEditorSupport { + return sqlEditor({ + autocomplete: { + defaultKeymap: false, + externalSources: [ + variableCompletionSource, + keywordCompletionSource, + ], + infoResolver, + }, + initialContext: contextFor(route), + initialEmbeddedRegions: pythonTemplateRegions(initialText), + service: sharedSqlService, + statementGutter: { + hideWhenNotFocused: true, + showInactive: true, + }, + }); +} + +interface MarimoSqlMetadata { + readonly route: VnextCompletionRoute; +} + +declare const setMarimoSqlMetadata: + StateEffectType; +declare const reconfigureSqlDialect: + StateEffectType; + +/** + * Marimo owns this transaction seam. `resultingText` is the document after + * `changes`; all interpretation inputs are emitted in the same transaction. + */ +function atomicSqlInputTransaction( + support: SqlEditorSupport, + input: { + readonly changes?: ChangeSpec; + readonly metadata: MarimoSqlMetadata; + readonly resultingText: string; + }, +): TransactionSpec { + const { route } = input.metadata; + const transaction: TransactionSpec = { + effects: [ + setMarimoSqlMetadata.of(input.metadata), + support.contextEffect.of(contextFor(route)), + reconfigureSqlDialect.of(route.dialect), + support.embeddedRegionsEffect.of( + pythonTemplateRegions(input.resultingText), + ), + ], + }; + return input.changes === undefined + ? transaction + : { ...transaction, changes: input.changes }; +} + +declare const duckdbRoute: VnextCompletionRoute; +declare const secondDuckdbRoute: VnextCompletionRoute; +declare const mysqlConnection: MarimoConnection & { + readonly dialect: "mysql"; +}; + +const firstSupport = createVnextSupport( + duckdbRoute, + "SELECT * FROM {df}", +); +const secondSupport = createVnextSupport( + secondDuckdbRoute, + "SELECT * FROM users", +); +const atomicSwitch = atomicSqlInputTransaction(firstSupport, { + changes: { from: 14, insert: "orders", to: 18 }, + metadata: { route: secondDuckdbRoute }, + resultingText: "SELECT * FROM orders", +}); +const legacyRoute = completionRoute(mysqlConnection); +if (legacyRoute.kind === "legacy") { + const source: CompletionSource = legacyRoute.source; + void source; +} + +const relationPath = [ + { quoted: false, role: "catalog", value: "memory" }, + { quoted: false, role: "schema", value: "main" }, + { quoted: false, role: "relation", value: "users" }, +] satisfies SqlCanonicalRelationPath; + +void atomicSwitch; +void firstSupport; +void marimoCatalogProvider; +void relationPath; +void secondSupport; + +// The application, not either editor support, owns the shared service. +sharedSqlService.dispose(); From b37641683fc0bcc6ba4d6ddc3ea2539b58c1b7ea Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:33:59 +0800 Subject: [PATCH 03/25] feat(vnext): add internal query binding model --- docs/adr/0006-query-binding-model.md | 102 ++ .../__tests__/query-binding-model.bench.ts | 89 ++ .../__tests__/query-binding-model.test.ts | 1062 +++++++++++++++ src/vnext/query-binding-model.ts | 1146 +++++++++++++++++ 4 files changed, 2399 insertions(+) create mode 100644 docs/adr/0006-query-binding-model.md create mode 100644 src/vnext/__tests__/query-binding-model.bench.ts create mode 100644 src/vnext/__tests__/query-binding-model.test.ts create mode 100644 src/vnext/query-binding-model.ts diff --git a/docs/adr/0006-query-binding-model.md b/docs/adr/0006-query-binding-model.md new file mode 100644 index 0000000..64271af --- /dev/null +++ b/docs/adr/0006-query-binding-model.md @@ -0,0 +1,102 @@ +# ADR 0006: Internal query binding model + +## Status + +Accepted for the vNext implementation. The model remains package-internal until a +column-completion vertical slice has validated it across the supported dialect +corpora. + +## Context + +Column completion needs more than a parser's flat table list. It must distinguish +query blocks, relation declarations, aliases, correlation, and clause-specific +visibility. The parser adapter also has strict trust boundaries: raw parser ASTs +are private, parser results can be partial, and cached analysis must belong to +the exact statement and parser authority that produced it. + +The model must remain cheap enough for interactive use and must not make the +existing parser-independent relation completion path wait for parsing. + +## Decision + +Add an internal, backend-neutral query binding model with: + +- statement-relative UTF-16 ranges; +- query blocks and relation bindings identified by model-local array indexes; +- a persistent scope chain in which each node adds at most one binding; +- ordered, globally non-overlapping visibility regions that point to a scope; +- independent query-block, relation-binding, and visibility coverage; +- closed issue and unknown-source reason sets; +- private authentication and exact statement/parser-authority ownership. + +Globally non-overlapping regions deliberately exclude nested query text from +their parent region. A decoder splits a parent region around nested queries. +This permits binary-search lookup without duplicating the visible binding set. +Persistent scopes then reconstruct bindings in declaration order using linear +space. In particular: + +- a `SELECT` list can point to the completed `FROM` scope even though it appears + earlier in the source; +- a join condition points to the scope containing prior relations and its + current right-hand relation; +- a correlated child block can enter through a parent scope; +- an ordinary derived table enters through a scope that excludes same-level + siblings; a future proven `LATERAL` decoder may opt into that scope. + +Aliases are the visible qualifier when present and hide a named relation's base +name. Identifier equality is supplied by the dialect runtime; the model does +not lowercase or otherwise reinterpret identifiers. + +Construction accepts untrusted plain data and validates it before issuing an +immutable authenticated model. It rejects accessors, sparse or oversized +arrays, malformed UTF-16 identifiers, invalid or unrelated ranges, forward +parent edges, repeated bindings in a scope chain, overlapping regions, and +unknown bindings paired with complete relation coverage. + +Resolution returns: + +- a visible binding list with complete or partial coverage; +- one resolved binding, known ambiguity, or authoritative no-match; +- unavailable instead of no-match when coverage is partial. + +## Ownership and invalidation + +The future parser service owns bounded parse/model caching and in-flight +deduplication. A document session owns the mapping from its current statement +slots to immutable models. The cache key includes exact statement text, +dialect, parser authority/conformance, and implementation version. +Statement-relative coordinates allow an unchanged statement to be reused after +an edit shifts its absolute document position. Source, dialect, or parser +authority changes invalidate the model; catalog epoch and completion context +changes do not. + +The interactive statement limit is initially 16 KiB. The model also caps query +blocks at 256, relation bindings at 1,024, scopes and regions at 2,048 each, +nesting by the block limit, issues at 256, path depth at 8, and identifier +components at 256 UTF-16 code units. + +## Degradation + +A decoder may claim complete evidence only for directly conforming parser output +and constructs it understands. Compatibility parses can supply useful partial +evidence but cannot prove absence. Unknown AST subtrees become an unknown source +or a closed issue and downgrade the relevant coverage; they are never silently +dropped under complete coverage. + +Invalid, failed, opaque, incomplete, or resource-limited analysis becomes +partial or unavailable. Parser-independent relation completion remains +available. Dremio remains partial or unavailable until it has an owned semantic +corpus. + +## Consequences + +The model can support lazy batched column loading without exposing a parser AST +or CodeMirror's eager `SQLNamespace`. Its indexes are not stable identities +across revisions. Projection inference, output columns, star expansion, types, +expression binding, semantic diagnostics, navigation, rename, formatting, and +vendor constructs such as `PIVOT`, `UNNEST`, table functions, and unproven +`LATERAL` semantics are explicitly outside this slice. + +The next slice is a strict AST-to-plain-data decoder, normalized inside the +worker, with direct-versus-worker differential tests. Public APIs and column +providers follow only after that decoder proves the model. diff --git a/src/vnext/__tests__/query-binding-model.bench.ts b/src/vnext/__tests__/query-binding-model.bench.ts new file mode 100644 index 0000000..0231e49 --- /dev/null +++ b/src/vnext/__tests__/query-binding-model.bench.ts @@ -0,0 +1,89 @@ +import { bench, describe } from "vitest"; +import { + createSqlQueryBindingModel, + resolveSqlRelationQualifier, + visibleSqlRelationBindingsAt, +} from "../query-binding-model.js"; +import type { SqlIdentifierComponent } from "../types.js"; + +const BINDING_COUNT = 1_000; +const TEXT = "x".repeat(10 * 1_024); +const AUTHORITY = {}; + +function component(value: string): SqlIdentifierComponent { + return { quoted: false, value }; +} + +const bindings = Array.from({ length: BINDING_COUNT }, (_, index) => ({ + alias: null, + owner: 0, + range: { from: index, to: index + 1 }, + source: { + kind: "named", + path: [component(`relation_${index}`)], + }, +})); + +const scopes = [ + { addedBinding: null, parentScope: null }, + ...bindings.map((_binding, index) => ({ + addedBinding: index, + parentScope: index, + })), +]; + +const input = { + bindings, + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: { from: 0, to: TEXT.length }, + }, + ], + coverage: { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }, + issues: [], + regions: [ + { + block: 0, + kind: "select-list", + range: { from: 0, to: TEXT.length }, + scope: BINDING_COUNT, + }, + ], + scopes, + statementRange: { from: 0, to: TEXT.length }, +}; + +const model = createSqlQueryBindingModel(TEXT, AUTHORITY, input); + +function asciiEqual( + left: SqlIdentifierComponent, + right: SqlIdentifierComponent, +): boolean { + return left.value === right.value; +} + +describe("query binding model", () => { + bench("validate 1,000 bindings in a 10 KiB statement", () => { + createSqlQueryBindingModel(TEXT, AUTHORITY, input); + }); + + bench("walk 1,000 visible bindings", () => { + visibleSqlRelationBindingsAt(model, 5_000); + }); + + bench("resolve the last of 1,000 qualifiers", () => { + resolveSqlRelationQualifier( + model, + 5_000, + component("relation_999"), + asciiEqual, + ); + }); +}); diff --git a/src/vnext/__tests__/query-binding-model.test.ts b/src/vnext/__tests__/query-binding-model.test.ts new file mode 100644 index 0000000..dee04a1 --- /dev/null +++ b/src/vnext/__tests__/query-binding-model.test.ts @@ -0,0 +1,1062 @@ +import { describe, expect, it } from "vitest"; +import { + createSqlQueryBindingModel, + isSqlQueryBindingModel, + isSqlQueryBindingModelError, + MAX_QUERY_BINDING_BLOCKS, + resolveSqlRelationQualifier, + sqlQueryBindingModelMatches, + visibleSqlRelationBindingsAt, + type SqlQueryBindingModel, + type SqlQueryBindingModelErrorCode, +} from "../query-binding-model.js"; +import type { SqlIdentifierComponent } from "../types.js"; + +const SQL = + "SELECT a.id FROM users AS a JOIN teams t ON a.team_id=t.id WHERE a.id>0"; + +function component(value: string, quoted = false) { + return { quoted, value }; +} + +function range(from: number, to: number) { + return { from, to }; +} + +function completeCoverage() { + return { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }; +} + +function fixture(overrides: Readonly> = {}) { + return { + bindings: [ + { + alias: { + explicit: true, + name: component("a"), + range: range(26, 27), + }, + owner: 0, + range: range(17, 27), + source: { + kind: "named", + path: [component("users")], + }, + }, + { + alias: { + explicit: false, + name: component("t"), + range: range(39, 40), + }, + owner: 0, + range: range(33, 40), + source: { + kind: "named", + path: [component("public"), component("teams")], + }, + }, + ], + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(0, SQL.length), + }, + ], + coverage: completeCoverage(), + issues: [], + regions: [ + { + block: 0, + kind: "select-list", + range: range(7, 11), + scope: 2, + }, + { + block: 0, + kind: "join-condition", + range: range(44, 59), + scope: 2, + }, + { + block: 0, + kind: "where", + range: range(66, SQL.length), + scope: 2, + }, + ], + scopes: [ + { addedBinding: null, parentScope: null }, + { addedBinding: 0, parentScope: 0 }, + { addedBinding: 1, parentScope: 1 }, + ], + statementRange: range(0, SQL.length), + ...overrides, + }; +} + +function model( + overrides: Readonly> = {}, + authority: object = {}, +): SqlQueryBindingModel { + return createSqlQueryBindingModel(SQL, authority, fixture(overrides)); +} + +function asciiEqual( + left: SqlIdentifierComponent, + right: SqlIdentifierComponent, +): boolean { + if (left.quoted || right.quoted) { + return left.quoted === right.quoted && left.value === right.value; + } + return left.value.toLowerCase() === right.value.toLowerCase(); +} + +function expectError( + operation: () => unknown, + code: SqlQueryBindingModelErrorCode, +): void { + try { + operation(); + } catch (error) { + expect(isSqlQueryBindingModelError(error)).toBe(true); + if (isSqlQueryBindingModelError(error)) { + expect(error.code).toBe(code); + } + return; + } + throw new Error("Expected a query binding model error"); +} + +describe("query binding model authentication", () => { + it("creates a deeply immutable model bound to exact source and authority", () => { + const authority = {}; + const input = fixture(); + const result = createSqlQueryBindingModel(SQL, authority, input); + + expect(isSqlQueryBindingModel(result)).toBe(true); + expect(isSqlQueryBindingModel(input)).toBe(false); + expect(sqlQueryBindingModelMatches(result, SQL, authority)).toBe(true); + expect(sqlQueryBindingModelMatches(result, `${SQL} `, authority)).toBe(false); + expect(sqlQueryBindingModelMatches(result, SQL, {})).toBe(false); + expect(Object.isFrozen(result)).toBe(true); + expect(Object.isFrozen(result.blocks)).toBe(true); + expect(Object.isFrozen(result.blocks[0])).toBe(true); + expect(Object.isFrozen(result.bindings[0]?.source)).toBe(true); + expect(Object.isFrozen(result.bindings[0]?.alias?.name)).toBe(true); + }); + + it("does not authenticate structural imitations", () => { + const imitation = fixture(); + expect(isSqlQueryBindingModel(null)).toBe(false); + expect(isSqlQueryBindingModel(1)).toBe(false); + expect(isSqlQueryBindingModel(imitation)).toBe(false); + expect( + sqlQueryBindingModelMatches(imitation, SQL, {}), + ).toBe(false); + expect(isSqlQueryBindingModelError(new Error("no"))).toBe(false); + expect(sqlQueryBindingModelMatches(null, SQL, {})).toBe(false); + }); +}); + +describe("query binding visibility", () => { + it("walks persistent scopes in declaration order", () => { + const result = visibleSqlRelationBindingsAt(model(), 8); + expect(result.status).toBe("ready"); + if (result.status === "ready") { + expect(result.region.kind).toBe("select-list"); + expect( + result.bindings.map((binding) => binding.alias?.name.value), + ).toEqual(["a", "t"]); + expect(result.coverage).toBe("complete"); + expect(result.issues).toEqual([]); + } + }); + + it("uses half-open regions and rejects invalid requests", () => { + const authenticated = model(); + expect(visibleSqlRelationBindingsAt(authenticated, 11)).toEqual({ + reason: "outside-visibility-region", + status: "unavailable", + }); + expect(visibleSqlRelationBindingsAt(authenticated, 44).status).toBe("ready"); + expect(visibleSqlRelationBindingsAt(authenticated, SQL.length)).toEqual({ + reason: "outside-visibility-region", + status: "unavailable", + }); + expect(visibleSqlRelationBindingsAt(authenticated, -1)).toEqual({ + reason: "invalid-model", + status: "unavailable", + }); + expect(visibleSqlRelationBindingsAt(authenticated, 1.5)).toEqual({ + reason: "invalid-model", + status: "unavailable", + }); + expect( + visibleSqlRelationBindingsAt(fixture(), 8), + ).toEqual({ reason: "invalid-model", status: "unavailable" }); + }); + + it("combines relation and visibility coverage", () => { + for (const coverage of [ + { + queryBlocks: "partial", + relationBindings: "complete", + visibility: "complete", + }, + { + queryBlocks: "complete", + relationBindings: "partial", + visibility: "complete", + }, + { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "partial", + }, + ]) { + const result = visibleSqlRelationBindingsAt(model({ coverage }), 8); + expect(result.status === "ready" && result.coverage).toBe( + coverage.queryBlocks === "partial" ? "complete" : "partial", + ); + } + }); +}); + +describe("query binding qualifier resolution", () => { + it("resolves aliases and makes an alias hide the base name", () => { + const authenticated = model(); + const alias = resolveSqlRelationQualifier( + authenticated, + 8, + component("A"), + asciiEqual, + ); + expect(alias.status).toBe("resolved"); + if (alias.status === "resolved") { + expect(alias.binding.alias?.name.value).toBe("a"); + expect(alias.coverage).toBe("complete"); + } + expect( + resolveSqlRelationQualifier( + authenticated, + 8, + component("users"), + asciiEqual, + ), + ).toEqual({ status: "no-match" }); + expect( + resolveSqlRelationQualifier( + authenticated, + 8, + component("TEAMS"), + asciiEqual, + ).status, + ).toBe("no-match"); + }); + + it("uses the last named path component and a CTE name without aliases", () => { + const bindings = [ + { + alias: null, + owner: 0, + range: range(17, 22), + source: { + kind: "named", + path: [component("public"), component("users")], + }, + }, + { + alias: null, + owner: 0, + range: range(33, 40), + source: { + declarationRange: range(0, 4), + kind: "cte", + name: component("teams"), + }, + }, + ]; + const authenticated = model({ bindings }); + expect( + resolveSqlRelationQualifier( + authenticated, + 8, + component("users"), + asciiEqual, + ).status, + ).toBe("resolved"); + expect( + resolveSqlRelationQualifier( + authenticated, + 8, + component("teams"), + asciiEqual, + ).status, + ).toBe("resolved"); + }); + + it("reports ambiguity and partial evidence explicitly", () => { + const bindings = [ + fixture().bindings[0], + { + ...fixture().bindings[1], + alias: { + explicit: false, + name: component("a"), + range: range(39, 40), + }, + }, + ]; + const ambiguous = resolveSqlRelationQualifier( + model({ bindings }), + 8, + component("a"), + asciiEqual, + ); + expect(ambiguous.status).toBe("ambiguous"); + if (ambiguous.status === "ambiguous") { + expect(ambiguous.bindings).toHaveLength(2); + expect(Object.isFrozen(ambiguous.bindings)).toBe(true); + } + + const coverage = { + queryBlocks: "complete", + relationBindings: "partial", + visibility: "complete", + }; + const partial = model({ coverage }); + const found = resolveSqlRelationQualifier( + partial, + 8, + component("a"), + asciiEqual, + ); + expect(found.status === "resolved" && found.coverage).toBe("partial"); + expect( + resolveSqlRelationQualifier( + partial, + 8, + component("missing"), + asciiEqual, + ), + ).toEqual({ reason: "partial-coverage", status: "unavailable" }); + }); + + it("propagates unavailable visibility and ignores unqualified derived sources", () => { + const blocks = [ + fixture().blocks[0], + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(17, 22), + }, + ]; + const bindings = [ + { + alias: null, + owner: 0, + range: range(17, 22), + source: { block: 1, kind: "derived" }, + }, + fixture().bindings[1], + ]; + const authenticated = model({ bindings, blocks }); + expect( + resolveSqlRelationQualifier( + authenticated, + 12, + component("t"), + asciiEqual, + ), + ).toEqual({ + reason: "outside-visibility-region", + status: "unavailable", + }); + expect( + resolveSqlRelationQualifier( + authenticated, + 8, + component("anything"), + asciiEqual, + ), + ).toEqual({ status: "no-match" }); + }); +}); + +describe("query binding model validation", () => { + it("rejects invalid source and authority inputs", () => { + expectError( + () => createSqlQueryBindingModel(1, {}, fixture()), + "invalid-source", + ); + expectError( + () => createSqlQueryBindingModel("", {}, fixture()), + "invalid-source", + ); + expectError( + () => + createSqlQueryBindingModel( + "x".repeat(16 * 1024 + 1), + {}, + fixture(), + ), + "resource-limit", + ); + expectError( + () => + createSqlQueryBindingModel( + SQL, + null, + fixture(), + ), + "invalid-authority", + ); + }); + + it.each([ + ["null input", null], + ["missing fields", {}], + ["no blocks", fixture({ blocks: [] })], + ["no scopes", fixture({ scopes: [] })], + [ + "wrong statement extent", + fixture({ statementRange: range(0, SQL.length - 1) }), + ], + [ + "empty statement extent", + fixture({ statementRange: range(0, 0) }), + ], + [ + "unsupported coverage", + fixture({ + coverage: { + ...completeCoverage(), + visibility: "unknown", + }, + }), + ], + ["non-array blocks", fixture({ blocks: {} })], + ])("rejects %s", (_name, input) => { + expectError( + () => createSqlQueryBindingModel(SQL, {}, input), + "invalid-model", + ); + }); + + it("rejects resource excess and sparse arrays", () => { + expectError( + () => + createSqlQueryBindingModel( + SQL, + {}, + fixture({ + blocks: Array.from( + { length: MAX_QUERY_BINDING_BLOCKS + 1 }, + () => fixture().blocks[0], + ), + }), + ), + "resource-limit", + ); + const sparse: unknown[] = []; + sparse.length = 1; + expectError( + () => + createSqlQueryBindingModel( + SQL, + {}, + fixture({ blocks: sparse }), + ), + "invalid-model", + ); + }); + + it("never invokes accessors", () => { + let calls = 0; + const hostile = fixture(); + Object.defineProperty(hostile, "blocks", { + enumerable: true, + get() { + calls += 1; + return []; + }, + }); + expectError( + () => createSqlQueryBindingModel(SQL, {}, hostile), + "invalid-model", + ); + expect(calls).toBe(0); + }); + + it("rejects malformed primitive fields without coercion", () => { + const invalidCases: readonly Readonly>[] = [ + { + bindings: [ + { + ...fixture().bindings[0], + alias: { + ...fixture().bindings[0]?.alias, + explicit: "yes", + }, + }, + fixture().bindings[1], + ], + }, + { + blocks: [ + { ...fixture().blocks[0], kind: 1 }, + ], + }, + { + blocks: [ + { ...fixture().blocks[0], baseScope: -1 }, + ], + }, + { + blocks: [ + { ...fixture().blocks[0], baseScope: 0.5 }, + ], + }, + { + regions: [ + { + ...fixture().regions[0], + range: range(11, 7), + }, + ], + }, + { + regions: [ + { + ...fixture().regions[0], + range: range(7, 7), + }, + ], + }, + ]; + for (const invalidCase of invalidCases) { + expectError(() => model(invalidCase), "invalid-model"); + } + }); + + it("rejects arrays whose length cannot be inspected as data", () => { + const hostileArray = new Proxy([], { + getOwnPropertyDescriptor(target, key) { + if (key === "length") { + return { + configurable: false, + enumerable: false, + value: "one", + writable: true, + }; + } + return Reflect.getOwnPropertyDescriptor(target, key); + }, + }); + expectError( + () => model({ blocks: hostileArray }), + "invalid-model", + ); + }); + + it("normalizes proxy and inspection failures", () => { + const hostile = new Proxy(fixture(), { + getOwnPropertyDescriptor() { + throw new Error("hostile trap"); + }, + }); + expectError( + () => createSqlQueryBindingModel(SQL, {}, hostile), + "invalid-model", + ); + }); + + it.each([ + [ + "forward block parent", + { + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: 1, + range: range(0, SQL.length), + }, + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(0, SQL.length), + }, + ], + }, + ], + [ + "child outside parent", + { + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(0, 20), + }, + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(19, 30), + }, + ], + }, + ], + [ + "forward scope parent", + { + scopes: [ + { addedBinding: null, parentScope: 1 }, + { addedBinding: null, parentScope: null }, + ], + }, + ], + [ + "repeated binding in scope chain", + { + scopes: [ + { addedBinding: 0, parentScope: null }, + { addedBinding: 0, parentScope: 0 }, + { addedBinding: 1, parentScope: 1 }, + ], + }, + ], + [ + "overlapping visibility", + { + regions: [ + { + block: 0, + kind: "select-list", + range: range(7, 20), + scope: 2, + }, + { + block: 0, + kind: "where", + range: range(19, 22), + scope: 2, + }, + ], + }, + ], + [ + "unordered visibility", + { + regions: [ + { + block: 0, + kind: "where", + range: range(66, 70), + scope: 2, + }, + { + block: 0, + kind: "select-list", + range: range(7, 11), + scope: 2, + }, + ], + }, + ], + ])("rejects %s", (_name, overrides) => { + expectError( + () => model(overrides), + "invalid-model", + ); + }); + + it("rejects invalid binding ownership and source relationships", () => { + const outside = [ + { + ...fixture().bindings[0], + range: range(0, SQL.length), + }, + fixture().bindings[1], + ]; + const narrowBlock = [ + { + ...fixture().blocks[0], + range: range(5, SQL.length), + }, + ]; + expectError(() => model({ bindings: outside, blocks: narrowBlock }), "invalid-model"); + + const blocks = [ + fixture().blocks[0], + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(17, 22), + }, + ]; + const bindings = [ + { + ...fixture().bindings[0], + source: { block: 1, kind: "derived" }, + }, + fixture().bindings[1], + ]; + expectError(() => model({ bindings, blocks }), "invalid-model"); + }); + + it("requires partial relation coverage for unknown sources", () => { + const bindings = [ + { + ...fixture().bindings[0], + source: { + kind: "unknown", + reason: "unsupported-relation-source", + }, + }, + fixture().bindings[1], + ]; + expectError(() => model({ bindings }), "invalid-model"); + const coverage = { + ...completeCoverage(), + relationBindings: "partial", + }; + expect(model({ bindings, coverage }).bindings[0]?.source.kind).toBe( + "unknown", + ); + }); + + it("rejects malformed identifiers, paths, ranges, aliases, enums, and indexes", () => { + const cases: readonly Readonly>[] = [ + { + bindings: [ + { + ...fixture().bindings[0], + source: { kind: "named", path: [] }, + }, + fixture().bindings[1], + ], + }, + { + bindings: [ + { + ...fixture().bindings[0], + source: { + kind: "named", + path: [component("\ud800")], + }, + }, + fixture().bindings[1], + ], + }, + { + bindings: [ + { + ...fixture().bindings[0], + alias: { + explicit: true, + name: component("a"), + range: range(28, 29), + }, + }, + fixture().bindings[1], + ], + }, + { + bindings: [ + { ...fixture().bindings[0], owner: 99 }, + fixture().bindings[1], + ], + }, + { + bindings: [ + { + ...fixture().bindings[0], + source: { kind: "mystery" }, + }, + fixture().bindings[1], + ], + }, + { + blocks: [ + { ...fixture().blocks[0], kind: "values" }, + ], + }, + { + regions: [ + { ...fixture().regions[0], kind: "window" }, + ], + }, + { + issues: [{ code: "mystery", range: range(0, 1) }], + }, + { + scopes: [ + { addedBinding: 99, parentScope: null }, + ], + }, + ]; + for (let index = 0; index < cases.length; index += 1) { + const overrides = cases[index]; + if (!overrides) { + throw new Error("Malformed fixture table contains a hole"); + } + expectError(() => model(overrides), "invalid-model"); + } + }); + + it("accepts every closed issue and unknown reason", () => { + const codes = [ + "ambiguous-alias", + "duplicate-alias", + "opaque-template-context", + "parser-compatibility", + "recursive-cte-uncertainty", + "resource-limit", + "unknown-correlation", + "unsupported-clause", + "unsupported-relation-source", + ]; + const coverage = { + queryBlocks: "partial", + relationBindings: "partial", + visibility: "partial", + }; + const issues = codes.map((code, index) => ({ + code, + range: range(index, index + 1), + })); + expect(model({ coverage, issues }).issues).toHaveLength(codes.length); + + for (const reason of [ + "opaque-template-context", + "unknown-correlation", + "unsupported-relation-source", + ]) { + const bindings = [ + { + ...fixture().bindings[0], + source: { kind: "unknown", reason }, + }, + fixture().bindings[1], + ]; + expect(model({ bindings, coverage }).bindings[0]?.source).toEqual({ + kind: "unknown", + reason, + }); + } + }); + + it("accepts well-formed surrogate pairs and rejects lone trailing surrogates", () => { + const validBindings = [ + { + ...fixture().bindings[0], + source: { + kind: "named", + path: [component("relation_\ud83d\ude80")], + }, + }, + fixture().bindings[1], + ]; + expect(model({ bindings: validBindings }).bindings[0]?.source.kind).toBe( + "named", + ); + const invalidBindings = [ + { + ...fixture().bindings[0], + source: { + kind: "named", + path: [component("\udc00")], + }, + }, + fixture().bindings[1], + ]; + expectError(() => model({ bindings: invalidBindings }), "invalid-model"); + }); + + it("rejects a region assigned outside its query block", () => { + const blocks = [ + fixture().blocks[0], + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(17, 22), + }, + ]; + const regions = [ + { + block: 1, + kind: "select-list", + range: range(7, 11), + scope: 2, + }, + ]; + expectError(() => model({ blocks, regions }), "invalid-model"); + }); + + it("allows parent correlation but rejects local or sibling bindings in entry scopes", () => { + const correlatedBlocks = [ + fixture().blocks[0], + { + baseScope: 1, + kind: "select", + parentBlock: 0, + range: range(44, 59), + }, + ]; + const correlatedRegions = [ + { + block: 1, + kind: "where", + range: range(45, 50), + scope: 1, + }, + ]; + expect( + model({ + blocks: correlatedBlocks, + regions: correlatedRegions, + }).blocks[1]?.baseScope, + ).toBe(1); + + const localEntryBlocks = [ + { ...fixture().blocks[0], baseScope: 1 }, + ]; + expectError( + () => model({ blocks: localEntryBlocks }), + "invalid-model", + ); + + const siblingBlocks = [ + fixture().blocks[0], + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(17, 22), + }, + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(33, 40), + }, + ]; + const siblingBindings = [ + { + alias: null, + owner: 1, + range: range(17, 22), + source: { kind: "named", path: [component("left_child")] }, + }, + { + alias: null, + owner: 0, + range: range(33, 40), + source: { kind: "named", path: [component("root_relation")] }, + }, + ]; + const siblingScopes = [ + { addedBinding: null, parentScope: null }, + { addedBinding: 0, parentScope: 0 }, + { addedBinding: 1, parentScope: 0 }, + ]; + const siblingRegions = [ + { + block: 2, + kind: "select-list", + range: range(34, 38), + scope: 1, + }, + ]; + expectError( + () => + model({ + bindings: siblingBindings, + blocks: siblingBlocks, + regions: siblingRegions, + scopes: siblingScopes, + }), + "invalid-model", + ); + }); + + it("rejects overlapping sibling query blocks", () => { + const blocks = [ + fixture().blocks[0], + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(17, 30), + }, + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(20, 40), + }, + ]; + expectError(() => model({ blocks }), "invalid-model"); + }); + + it("preserves invariants across generated scope chains", () => { + for (let count = 1; count <= 32; count += 1) { + const text = "x".repeat(Math.max(count, 2)); + const bindings = Array.from({ length: count }, (_, index) => ({ + alias: null, + owner: 0, + range: range(index, index + 1), + source: { + kind: "named", + path: [component(`r${index}`)], + }, + })); + const scopes = [ + { addedBinding: null, parentScope: null }, + ...bindings.map((_binding, index) => ({ + addedBinding: index, + parentScope: index, + })), + ]; + const generated = createSqlQueryBindingModel(text, {}, { + bindings, + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(0, text.length), + }, + ], + coverage: completeCoverage(), + issues: [], + regions: [ + { + block: 0, + kind: "other", + range: range(0, text.length), + scope: count, + }, + ], + scopes, + statementRange: range(0, text.length), + }); + const visible = visibleSqlRelationBindingsAt(generated, 0); + expect(visible.status === "ready" && visible.bindings).toHaveLength(count); + } + }); +}); diff --git a/src/vnext/query-binding-model.ts b/src/vnext/query-binding-model.ts new file mode 100644 index 0000000..8aeb4a4 --- /dev/null +++ b/src/vnext/query-binding-model.ts @@ -0,0 +1,1146 @@ +import type { + SqlIdentifierComponent, + SqlIdentifierPath, + SqlTextRange, +} from "./types.js"; + +export const MAX_QUERY_BINDING_STATEMENT_LENGTH = 16 * 1024; +export const MAX_QUERY_BINDING_BLOCKS = 256; +export const MAX_QUERY_BINDINGS = 1_024; +export const MAX_QUERY_BINDING_SCOPES = 2_048; +export const MAX_QUERY_VISIBILITY_REGIONS = 2_048; +export const MAX_QUERY_BINDING_ISSUES = 256; +export const MAX_QUERY_BINDING_PATH_COMPONENTS = 8; +export const MAX_QUERY_BINDING_IDENTIFIER_LENGTH = 256; + +export type SqlQueryBindingCoverage = "complete" | "partial"; + +export interface SqlQueryBindingCoverageSet { + readonly queryBlocks: SqlQueryBindingCoverage; + readonly relationBindings: SqlQueryBindingCoverage; + readonly visibility: SqlQueryBindingCoverage; +} + +export type SqlQueryBindingIssueCode = + | "ambiguous-alias" + | "duplicate-alias" + | "opaque-template-context" + | "parser-compatibility" + | "recursive-cte-uncertainty" + | "resource-limit" + | "unknown-correlation" + | "unsupported-clause" + | "unsupported-relation-source"; + +export interface SqlQueryBindingIssue { + readonly code: SqlQueryBindingIssueCode; + readonly range: SqlTextRange; +} + +export type SqlQueryBlockKind = "compound" | "select"; + +export interface SqlQueryBlock { + readonly baseScope: number; + readonly kind: SqlQueryBlockKind; + readonly parentBlock: number | null; + readonly range: SqlTextRange; +} + +export interface SqlRelationScope { + readonly addedBinding: number | null; + readonly parentScope: number | null; +} + +export type SqlVisibilityRegionKind = + | "from-source" + | "group-by" + | "having" + | "join-condition" + | "limit" + | "order-by" + | "other" + | "qualify" + | "select-list" + | "where"; + +export interface SqlVisibilityRegion { + readonly block: number; + readonly kind: SqlVisibilityRegionKind; + readonly range: SqlTextRange; + readonly scope: number; +} + +export interface SqlNamedRelationSource { + readonly kind: "named"; + readonly path: SqlIdentifierPath; +} + +export interface SqlDerivedRelationSource { + readonly block: number; + readonly kind: "derived"; +} + +export interface SqlCteRelationSource { + readonly declarationRange: SqlTextRange; + readonly kind: "cte"; + readonly name: SqlIdentifierComponent; +} + +export interface SqlUnknownRelationSource { + readonly kind: "unknown"; + readonly reason: + | "opaque-template-context" + | "unknown-correlation" + | "unsupported-relation-source"; +} + +export type SqlRelationBindingSource = + | SqlCteRelationSource + | SqlDerivedRelationSource + | SqlNamedRelationSource + | SqlUnknownRelationSource; + +export interface SqlRelationBindingAlias { + readonly explicit: boolean; + readonly name: SqlIdentifierComponent; + readonly range: SqlTextRange; +} + +export interface SqlRelationBinding { + readonly alias: SqlRelationBindingAlias | null; + readonly owner: number; + readonly range: SqlTextRange; + readonly source: SqlRelationBindingSource; +} + +export interface SqlQueryBindingModel { + readonly blocks: readonly SqlQueryBlock[]; + readonly bindings: readonly SqlRelationBinding[]; + readonly coverage: SqlQueryBindingCoverageSet; + readonly issues: readonly SqlQueryBindingIssue[]; + readonly regions: readonly SqlVisibilityRegion[]; + readonly scopes: readonly SqlRelationScope[]; + readonly statementRange: SqlTextRange; +} + +export type SqlQueryBindingModelErrorCode = + | "invalid-authority" + | "invalid-model" + | "invalid-source" + | "resource-limit"; + +const modelErrors = new WeakSet(); +const models = new WeakSet(); +const modelOrigins = new WeakMap< + object, + { readonly authority: object; readonly statementText: string } +>(); + +export class SqlQueryBindingModelError extends Error { + readonly code: SqlQueryBindingModelErrorCode; + + constructor(code: SqlQueryBindingModelErrorCode, message: string) { + super(message); + this.name = "SqlQueryBindingModelError"; + this.code = code; + modelErrors.add(this); + } +} + +export function isSqlQueryBindingModelError( + candidate: unknown, +): candidate is SqlQueryBindingModelError { + return ( + candidate !== null && + typeof candidate === "object" && + modelErrors.has(candidate) + ); +} + +export function isSqlQueryBindingModel( + candidate: unknown, +): candidate is SqlQueryBindingModel { + return ( + candidate !== null && + typeof candidate === "object" && + models.has(candidate) + ); +} + +export function sqlQueryBindingModelMatches( + model: unknown, + statementText: string, + authority: unknown, +): boolean { + if (model === null || typeof model !== "object") { + return false; + } + const origin = modelOrigins.get(model); + return ( + origin !== undefined && + origin.authority === authority && + origin.statementText === statementText + ); +} + +interface FoundProperty { + readonly found: true; + readonly value: unknown; +} + +interface MissingProperty { + readonly found: false; +} + +type DataProperty = FoundProperty | MissingProperty; + +function ownDataProperty( + value: object, + key: PropertyKey, + subject: string, +): DataProperty { + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (descriptor === undefined) { + return { found: false }; + } + if (!("value" in descriptor)) { + throw invalid(`${subject}.${String(key)} cannot be an accessor`); + } + return { found: true, value: descriptor.value }; +} + +function requiredProperty( + value: object, + key: PropertyKey, + subject: string, +): unknown { + const property = ownDataProperty(value, key, subject); + if (!property.found) { + throw invalid(`${subject}.${String(key)} is required`); + } + return property.value; +} + +function objectValue(value: unknown, subject: string): object { + if (value === null || typeof value !== "object") { + throw invalid(`${subject} must be an object`); + } + return value; +} + +function invalid(message: string): SqlQueryBindingModelError { + return new SqlQueryBindingModelError("invalid-model", message); +} + +function resource(message: string): SqlQueryBindingModelError { + return new SqlQueryBindingModelError("resource-limit", message); +} + +function integer( + value: unknown, + subject: string, + maximum: number, +): number { + if ( + typeof value !== "number" || + !Number.isSafeInteger(value) || + value < 0 || + value >= maximum + ) { + throw invalid(`${subject} must be an in-bounds non-negative integer`); + } + return value; +} + +function nullableInteger( + value: unknown, + subject: string, + maximum: number, +): number | null { + return value === null ? null : integer(value, subject, maximum); +} + +function booleanValue(value: unknown, subject: string): boolean { + if (typeof value !== "boolean") { + throw invalid(`${subject} must be a boolean`); + } + return value; +} + +function enumValue( + value: unknown, + subject: string, + allowed: readonly Value[], +): Value { + if (typeof value !== "string") { + throw invalid(`${subject} is not supported`); + } + const match = allowed.find((candidate) => candidate === value); + if (match === undefined) { + throw invalid(`${subject} is not supported`); + } + return match; +} + +function arrayValues( + value: unknown, + subject: string, + maximum: number, +): readonly unknown[] { + if (!Array.isArray(value)) { + throw invalid(`${subject} must be an array`); + } + const lengthProperty = ownDataProperty(value, "length", subject); + if ( + !lengthProperty.found || + typeof lengthProperty.value !== "number" || + !Number.isSafeInteger(lengthProperty.value) || + lengthProperty.value < 0 + ) { + throw invalid(`${subject} has an invalid length`); + } + if (lengthProperty.value > maximum) { + throw resource(`${subject} exceeds its ${maximum} item limit`); + } + const result: unknown[] = []; + for (let index = 0; index < lengthProperty.value; index += 1) { + const item = ownDataProperty(value, index, subject); + if (!item.found) { + throw invalid(`${subject} cannot contain holes`); + } + result.push(item.value); + } + return result; +} + +function rangeValue( + value: unknown, + statementLength: number, + subject: string, + allowEmpty = true, +): SqlTextRange { + const object = objectValue(value, subject); + const from = integer( + requiredProperty(object, "from", subject), + `${subject}.from`, + statementLength + 1, + ); + const to = integer( + requiredProperty(object, "to", subject), + `${subject}.to`, + statementLength + 1, + ); + if (from > to || (!allowEmpty && from === to)) { + throw invalid(`${subject} must be a valid half-open range`); + } + return Object.freeze({ from, to }); +} + +function rangeContains( + outer: SqlTextRange, + inner: SqlTextRange, +): boolean { + return outer.from <= inner.from && inner.to <= outer.to; +} + +function authenticatedItem( + values: readonly Value[], + index: number, +): Value { + const value = values[index]; + if (value === undefined) { + throw new Error("Authenticated query binding model invariant was violated"); + } + return value; +} + +function isBlockAncestorOrSelf( + model: SqlQueryBindingModel, + ancestor: number, + block: number, +): boolean { + let cursor: number | null = block; + while (cursor !== null) { + if (cursor === ancestor) { + return true; + } + cursor = authenticatedItem(model.blocks, cursor).parentBlock; + } + return false; +} + +function validateScopeOwners( + model: SqlQueryBindingModel, + scopeIndex: number, + blockIndex: number, + includeLocal: boolean, + subject: string, +): void { + let cursor: number | null = scopeIndex; + while (cursor !== null) { + const scope: SqlRelationScope = + authenticatedItem(model.scopes, cursor); + if (scope.addedBinding !== null) { + const binding = authenticatedItem(model.bindings, scope.addedBinding); + const visible = + isBlockAncestorOrSelf(model, binding.owner, blockIndex) && + (includeLocal || binding.owner !== blockIndex); + if (!visible) { + throw invalid(`${subject} contains an unrelated relation binding`); + } + } + cursor = scope.parentScope; + } +} + +function wellFormedString(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const trailing = value.charCodeAt(index + 1); + if (!(trailing >= 0xdc00 && trailing <= 0xdfff)) { + return false; + } + index += 1; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return false; + } + } + return true; +} + +function identifierValue( + value: unknown, + subject: string, +): SqlIdentifierComponent { + const object = objectValue(value, subject); + const text = requiredProperty(object, "value", subject); + if ( + typeof text !== "string" || + text.length === 0 || + text.length > MAX_QUERY_BINDING_IDENTIFIER_LENGTH || + !wellFormedString(text) + ) { + throw invalid(`${subject}.value must be a bounded well-formed string`); + } + return Object.freeze({ + quoted: booleanValue( + requiredProperty(object, "quoted", subject), + `${subject}.quoted`, + ), + value: text, + }); +} + +function pathValue( + value: unknown, + subject: string, +): SqlIdentifierPath { + const input = arrayValues( + value, + subject, + MAX_QUERY_BINDING_PATH_COMPONENTS, + ); + if (input.length === 0) { + throw invalid(`${subject} cannot be empty`); + } + return Object.freeze( + input.map((component, index) => + identifierValue(component, `${subject}[${index}]`), + ), + ); +} + +const BLOCK_KINDS: readonly SqlQueryBlockKind[] = Object.freeze([ + "compound", + "select", +]); +const REGION_KINDS: readonly SqlVisibilityRegionKind[] = Object.freeze([ + "from-source", + "group-by", + "having", + "join-condition", + "limit", + "order-by", + "other", + "qualify", + "select-list", + "where", +]); +const ISSUE_CODES: readonly SqlQueryBindingIssueCode[] = Object.freeze([ + "ambiguous-alias", + "duplicate-alias", + "opaque-template-context", + "parser-compatibility", + "recursive-cte-uncertainty", + "resource-limit", + "unknown-correlation", + "unsupported-clause", + "unsupported-relation-source", +]); +const UNKNOWN_REASONS: readonly SqlUnknownRelationSource["reason"][] = + Object.freeze([ + "opaque-template-context", + "unknown-correlation", + "unsupported-relation-source", + ]); + +function coverageValue( + value: unknown, +): SqlQueryBindingCoverageSet { + const object = objectValue(value, "query binding coverage"); + const read = (key: string): SqlQueryBindingCoverage => { + const result = enumValue( + requiredProperty(object, key, "query binding coverage"), + `query binding coverage.${key}`, + ["complete", "partial"], + ); + return result === "complete" ? "complete" : "partial"; + }; + return Object.freeze({ + queryBlocks: read("queryBlocks"), + relationBindings: read("relationBindings"), + visibility: read("visibility"), + }); +} + +function blockValue( + value: unknown, + index: number, + count: number, + scopeCount: number, + statementLength: number, +): SqlQueryBlock { + const subject = `query block ${index}`; + const object = objectValue(value, subject); + const kind = enumValue( + requiredProperty(object, "kind", subject), + `${subject}.kind`, + BLOCK_KINDS, + ); + const parentBlock = nullableInteger( + requiredProperty(object, "parentBlock", subject), + `${subject}.parentBlock`, + count, + ); + if (parentBlock !== null && parentBlock >= index) { + throw invalid(`${subject} parent must precede its child`); + } + return Object.freeze({ + baseScope: integer( + requiredProperty(object, "baseScope", subject), + `${subject}.baseScope`, + scopeCount, + ), + kind: kind === "compound" ? "compound" : "select", + parentBlock, + range: rangeValue( + requiredProperty(object, "range", subject), + statementLength, + `${subject}.range`, + false, + ), + }); +} + +function scopeValue( + value: unknown, + index: number, + count: number, + bindingCount: number, +): SqlRelationScope { + const subject = `relation scope ${index}`; + const object = objectValue(value, subject); + const parentScope = nullableInteger( + requiredProperty(object, "parentScope", subject), + `${subject}.parentScope`, + count, + ); + if (parentScope !== null && parentScope >= index) { + throw invalid(`${subject} parent must precede its child`); + } + return Object.freeze({ + addedBinding: nullableInteger( + requiredProperty(object, "addedBinding", subject), + `${subject}.addedBinding`, + bindingCount, + ), + parentScope, + }); +} + +function aliasValue( + value: unknown, + statementLength: number, + subject: string, +): SqlRelationBindingAlias | null { + if (value === null) { + return null; + } + const object = objectValue(value, subject); + return Object.freeze({ + explicit: booleanValue( + requiredProperty(object, "explicit", subject), + `${subject}.explicit`, + ), + name: identifierValue( + requiredProperty(object, "name", subject), + `${subject}.name`, + ), + range: rangeValue( + requiredProperty(object, "range", subject), + statementLength, + `${subject}.range`, + false, + ), + }); +} + +function sourceValue( + value: unknown, + blockCount: number, + statementLength: number, + subject: string, +): SqlRelationBindingSource { + const object = objectValue(value, subject); + const kind = requiredProperty(object, "kind", subject); + if (kind === "named") { + return Object.freeze({ + kind, + path: pathValue(requiredProperty(object, "path", subject), `${subject}.path`), + }); + } + if (kind === "derived") { + return Object.freeze({ + block: integer( + requiredProperty(object, "block", subject), + `${subject}.block`, + blockCount, + ), + kind, + }); + } + if (kind === "cte") { + return Object.freeze({ + declarationRange: rangeValue( + requiredProperty(object, "declarationRange", subject), + statementLength, + `${subject}.declarationRange`, + false, + ), + kind, + name: identifierValue( + requiredProperty(object, "name", subject), + `${subject}.name`, + ), + }); + } + if (kind === "unknown") { + const reason = enumValue( + requiredProperty(object, "reason", subject), + `${subject}.reason`, + UNKNOWN_REASONS, + ); + if (reason === "opaque-template-context") { + return Object.freeze({ kind, reason }); + } + if (reason === "unknown-correlation") { + return Object.freeze({ kind, reason }); + } + return Object.freeze({ + kind, + reason: "unsupported-relation-source", + }); + } + throw invalid(`${subject}.kind is not supported`); +} + +function bindingValue( + value: unknown, + index: number, + blockCount: number, + statementLength: number, +): SqlRelationBinding { + const subject = `relation binding ${index}`; + const object = objectValue(value, subject); + const range = rangeValue( + requiredProperty(object, "range", subject), + statementLength, + `${subject}.range`, + false, + ); + const alias = aliasValue( + requiredProperty(object, "alias", subject), + statementLength, + `${subject}.alias`, + ); + if (alias !== null && !rangeContains(range, alias.range)) { + throw invalid(`${subject} must contain its alias range`); + } + return Object.freeze({ + alias, + owner: integer( + requiredProperty(object, "owner", subject), + `${subject}.owner`, + blockCount, + ), + range, + source: sourceValue( + requiredProperty(object, "source", subject), + blockCount, + statementLength, + `${subject}.source`, + ), + }); +} + +function regionValue( + value: unknown, + index: number, + blockCount: number, + scopeCount: number, + statementLength: number, +): SqlVisibilityRegion { + const subject = `visibility region ${index}`; + const object = objectValue(value, subject); + const kind = enumValue( + requiredProperty(object, "kind", subject), + `${subject}.kind`, + REGION_KINDS, + ); + return Object.freeze({ + block: integer( + requiredProperty(object, "block", subject), + `${subject}.block`, + blockCount, + ), + kind, + range: rangeValue( + requiredProperty(object, "range", subject), + statementLength, + `${subject}.range`, + false, + ), + scope: integer( + requiredProperty(object, "scope", subject), + `${subject}.scope`, + scopeCount, + ), + }); +} + +function issueValue( + value: unknown, + index: number, + statementLength: number, +): SqlQueryBindingIssue { + const subject = `query binding issue ${index}`; + const object = objectValue(value, subject); + const code = enumValue( + requiredProperty(object, "code", subject), + `${subject}.code`, + ISSUE_CODES, + ); + return Object.freeze({ + code, + range: rangeValue( + requiredProperty(object, "range", subject), + statementLength, + `${subject}.range`, + ), + }); +} + +function validateRelationships(model: SqlQueryBindingModel): void { + for (let index = 0; index < model.blocks.length; index += 1) { + const block = authenticatedItem(model.blocks, index); + if (block.parentBlock !== null) { + const parent = authenticatedItem(model.blocks, block.parentBlock); + if (!rangeContains(parent.range, block.range)) { + throw invalid(`query block ${index} is outside its parent`); + } + } + validateScopeOwners( + model, + block.baseScope, + index, + false, + `query block ${index} base scope`, + ); + for (let sibling = 0; sibling < index; sibling += 1) { + const other = authenticatedItem(model.blocks, sibling); + const overlaps = + block.range.from < other.range.to && + other.range.from < block.range.to; + if ( + overlaps && + !isBlockAncestorOrSelf(model, sibling, index) && + !isBlockAncestorOrSelf(model, index, sibling) + ) { + throw invalid(`query block ${index} overlaps an unrelated block`); + } + } + } + for (let index = 0; index < model.bindings.length; index += 1) { + const binding = authenticatedItem(model.bindings, index); + const owner = authenticatedItem(model.blocks, binding.owner); + if (!rangeContains(owner.range, binding.range)) { + throw invalid(`relation binding ${index} is outside its owner`); + } + if (binding.source.kind === "derived") { + const derived = authenticatedItem(model.blocks, binding.source.block); + if (derived.parentBlock !== binding.owner) { + throw invalid(`derived relation binding ${index} has an unrelated block`); + } + } + if ( + model.coverage.relationBindings === "complete" && + binding.source.kind === "unknown" + ) { + throw invalid("complete relation coverage cannot contain unknown bindings"); + } + } + const introducedBindings = new Set(); + for (let index = 0; index < model.scopes.length; index += 1) { + const scope = authenticatedItem(model.scopes, index); + if (scope.addedBinding !== null) { + if (introducedBindings.has(scope.addedBinding)) { + throw invalid(`relation binding ${scope.addedBinding} is introduced twice`); + } + introducedBindings.add(scope.addedBinding); + } + } + if (introducedBindings.size !== model.bindings.length) { + throw invalid("every relation binding must be introduced by exactly one scope"); + } + let previousEnd = 0; + for (let index = 0; index < model.regions.length; index += 1) { + const region = authenticatedItem(model.regions, index); + if (region.range.from < previousEnd) { + throw invalid("visibility regions must be ordered and non-overlapping"); + } + const block = authenticatedItem(model.blocks, region.block); + if (!rangeContains(block.range, region.range)) { + throw invalid(`visibility region ${index} is outside its block`); + } + validateScopeOwners( + model, + region.scope, + region.block, + true, + `visibility region ${index} scope`, + ); + previousEnd = region.range.to; + } +} + +export function createSqlQueryBindingModel( + statementText: unknown, + authority: unknown, + input: unknown, +): SqlQueryBindingModel { + if (typeof statementText !== "string") { + throw new SqlQueryBindingModelError( + "invalid-source", + "query binding statement text must be a string", + ); + } + if ( + statementText.length === 0 || + statementText.length > MAX_QUERY_BINDING_STATEMENT_LENGTH + ) { + throw new SqlQueryBindingModelError( + statementText.length > MAX_QUERY_BINDING_STATEMENT_LENGTH + ? "resource-limit" + : "invalid-source", + `query binding statement must contain 1-${MAX_QUERY_BINDING_STATEMENT_LENGTH} UTF-16 code units`, + ); + } + if (authority === null || typeof authority !== "object") { + throw new SqlQueryBindingModelError( + "invalid-authority", + "query binding parser authority must be an object identity", + ); + } + try { + const object = objectValue(input, "query binding model"); + const blockInputs = arrayValues( + requiredProperty(object, "blocks", "query binding model"), + "query blocks", + MAX_QUERY_BINDING_BLOCKS, + ); + const bindingInputs = arrayValues( + requiredProperty(object, "bindings", "query binding model"), + "relation bindings", + MAX_QUERY_BINDINGS, + ); + const scopeInputs = arrayValues( + requiredProperty(object, "scopes", "query binding model"), + "relation scopes", + MAX_QUERY_BINDING_SCOPES, + ); + const regionInputs = arrayValues( + requiredProperty(object, "regions", "query binding model"), + "visibility regions", + MAX_QUERY_VISIBILITY_REGIONS, + ); + const issueInputs = arrayValues( + requiredProperty(object, "issues", "query binding model"), + "query binding issues", + MAX_QUERY_BINDING_ISSUES, + ); + if (blockInputs.length === 0 || scopeInputs.length === 0) { + throw invalid("query binding model requires a block and a scope"); + } + const statementRange = rangeValue( + requiredProperty(object, "statementRange", "query binding model"), + statementText.length, + "query binding model.statementRange", + false, + ); + if (statementRange.from !== 0 || statementRange.to !== statementText.length) { + throw invalid("statementRange must cover the exact statement text"); + } + const blocks = Object.freeze( + blockInputs.map((block, index) => + blockValue( + block, + index, + blockInputs.length, + scopeInputs.length, + statementText.length, + ), + ), + ); + const bindings = Object.freeze( + bindingInputs.map((binding, index) => + bindingValue( + binding, + index, + blockInputs.length, + statementText.length, + ), + ), + ); + const scopes = Object.freeze( + scopeInputs.map((scope, index) => + scopeValue( + scope, + index, + scopeInputs.length, + bindingInputs.length, + ), + ), + ); + const regions = Object.freeze( + regionInputs.map((region, index) => + regionValue( + region, + index, + blockInputs.length, + scopeInputs.length, + statementText.length, + ), + ), + ); + const issues = Object.freeze( + issueInputs.map((issue, index) => + issueValue(issue, index, statementText.length), + ), + ); + const model: SqlQueryBindingModel = Object.freeze({ + blocks, + bindings, + coverage: coverageValue( + requiredProperty(object, "coverage", "query binding model"), + ), + issues, + regions, + scopes, + statementRange, + }); + validateRelationships(model); + models.add(model); + modelOrigins.set(model, Object.freeze({ authority, statementText })); + return model; + } catch (error) { + if (isSqlQueryBindingModelError(error)) { + throw error; + } + throw new SqlQueryBindingModelError( + "invalid-model", + "query binding model could not be inspected safely", + ); + } +} + +export interface SqlVisibleRelationBindings { + readonly bindings: readonly SqlRelationBinding[]; + readonly coverage: SqlQueryBindingCoverage; + readonly issues: readonly SqlQueryBindingIssue[]; + readonly region: SqlVisibilityRegion; + readonly status: "ready"; +} + +export interface SqlVisibleRelationBindingsUnavailable { + readonly reason: "invalid-model" | "outside-visibility-region"; + readonly status: "unavailable"; +} + +export type SqlVisibleRelationBindingsResult = + | SqlVisibleRelationBindings + | SqlVisibleRelationBindingsUnavailable; + +function regionAt( + regions: readonly SqlVisibilityRegion[], + position: number, +): SqlVisibilityRegion | null { + let low = 0; + let high = regions.length; + while (low < high) { + const middle = low + Math.floor((high - low) / 2); + const region = regions[middle]; + if (!region || region.range.from > position) { + high = middle; + } else { + low = middle + 1; + } + } + const candidate = regions[low - 1]; + return candidate && position < candidate.range.to ? candidate : null; +} + +export function visibleSqlRelationBindingsAt( + model: unknown, + position: number, +): SqlVisibleRelationBindingsResult { + if ( + !isSqlQueryBindingModel(model) || + !Number.isSafeInteger(position) || + position < 0 || + position > model.statementRange.to + ) { + return Object.freeze({ reason: "invalid-model", status: "unavailable" }); + } + const region = regionAt(model.regions, position); + if (region === null) { + return Object.freeze({ + reason: "outside-visibility-region", + status: "unavailable", + }); + } + const reversed: SqlRelationBinding[] = []; + let cursor: number | null = region.scope; + while (cursor !== null) { + const scope: SqlRelationScope = + authenticatedItem(model.scopes, cursor); + if (scope.addedBinding !== null) { + const binding = authenticatedItem(model.bindings, scope.addedBinding); + reversed.push(binding); + } + cursor = scope.parentScope; + } + reversed.reverse(); + const coverage = + model.coverage.relationBindings === "complete" && + model.coverage.visibility === "complete" + ? "complete" + : "partial"; + return Object.freeze({ + bindings: Object.freeze(reversed), + coverage, + issues: model.issues, + region, + status: "ready", + }); +} + +export type SqlIdentifierComparator = ( + left: SqlIdentifierComponent, + right: SqlIdentifierComponent, +) => boolean; + +export interface SqlQualifierResolutionFound { + readonly binding: SqlRelationBinding; + readonly coverage: SqlQueryBindingCoverage; + readonly status: "resolved"; +} + +export interface SqlQualifierResolutionAmbiguous { + readonly bindings: readonly SqlRelationBinding[]; + readonly coverage: SqlQueryBindingCoverage; + readonly status: "ambiguous"; +} + +export interface SqlQualifierResolutionMissing { + readonly status: "no-match"; +} + +export interface SqlQualifierResolutionUnavailable { + readonly reason: + | "invalid-model" + | "outside-visibility-region" + | "partial-coverage"; + readonly status: "unavailable"; +} + +export type SqlQualifierResolution = + | SqlQualifierResolutionAmbiguous + | SqlQualifierResolutionFound + | SqlQualifierResolutionMissing + | SqlQualifierResolutionUnavailable; + +function bindingVisibleName( + binding: SqlRelationBinding, +): SqlIdentifierComponent | null { + if (binding.alias !== null) { + return binding.alias.name; + } + if (binding.source.kind === "named") { + return binding.source.path[binding.source.path.length - 1] ?? null; + } + if (binding.source.kind === "cte") { + return binding.source.name; + } + return null; +} + +export function resolveSqlRelationQualifier( + model: unknown, + position: number, + qualifier: SqlIdentifierComponent, + equals: SqlIdentifierComparator, +): SqlQualifierResolution { + const visible = visibleSqlRelationBindingsAt(model, position); + if (visible.status === "unavailable") { + return visible; + } + const matches: SqlRelationBinding[] = []; + for (const binding of visible.bindings) { + const name = bindingVisibleName(binding); + if (name !== null && equals(name, qualifier)) { + matches.push(binding); + } + } + if (matches.length === 0) { + return visible.coverage === "complete" + ? Object.freeze({ status: "no-match" }) + : Object.freeze({ + reason: "partial-coverage", + status: "unavailable", + }); + } + if (matches.length === 1) { + const binding = authenticatedItem(matches, 0); + return Object.freeze({ + binding, + coverage: visible.coverage, + status: "resolved", + }); + } + return Object.freeze({ + bindings: Object.freeze(matches), + coverage: visible.coverage, + status: "ambiguous", + }); +} From d44f57cab34ea5f9817f2177ed2f659343438efb Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:39:29 +0800 Subject: [PATCH 04/25] feat(vnext): gate SQL completion positions --- docs/vnext/marimo-sql-migration.md | 28 ++-- .../codemirror/__tests__/sql-editor.test.ts | 138 ++++++++++++++++ .../sql-editor-completion-gate.test.ts | 153 ++++++++++++++++++ src/vnext/codemirror/sql-editor.ts | 45 ++++++ .../marimo-sql-migration.test-d.ts | 34 ++++ 5 files changed, 385 insertions(+), 13 deletions(-) create mode 100644 src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts diff --git a/docs/vnext/marimo-sql-migration.md b/docs/vnext/marimo-sql-migration.md index 7cb7285..09afc39 100644 --- a/docs/vnext/marimo-sql-migration.md +++ b/docs/vnext/marimo-sql-migration.md @@ -58,18 +58,19 @@ The region scanner recognizes single `{python}` expressions, ignores escaped set after every document change. SQL completion is unavailable inside those regions while the external variable source remains active. -There is one unresolved insertion-point edge case. Embedded regions are -non-empty half-open ranges. An unmatched `{` ending at the document boundary -cannot contain the cursor position at `doc.length`, so regions alone cannot -suppress SQL completion at that exact point. Before cutover, either: - -- add an adapter completion gate for host-owned embedded insertion points; -- extend the core template contract with an explicit open-ended cursor - barrier; or -- keep a routed completion source that returns only variable results for the - unmatched-expression site. - -Do not encode a range beyond the document or an empty region. +Embedded regions are non-empty half-open ranges, so an unmatched `{` ending at +the document boundary cannot contain the insertion point at `doc.length`. +Marimo closes that boundary with +`autocomplete.isCompletionPositionAllowed`. Its synchronous scanner returns +false while the insertion point is in an unmatched Python expression. The +adapter then skips the SQL session source, while still running the installed +variable and keyword sources. A false result or thrown gate is fail-closed, +and a gate transition to false cancels owned SQL completion work and disposes +owned rich-info resources. + +The gate receives only the immutable CodeMirror `EditorState` and numeric +position. It must remain synchronous and side-effect free. Do not encode a +range beyond the document or an empty embedded region. The rich-info resolver uses catalog provenance IDs to look up current marimo metadata, mounts React into a new element, and returns @@ -119,7 +120,8 @@ provider request per relation. 1. Capture golden legacy results for labels, kinds, edit ranges, qualification, and details across representative connection shapes. 2. Land the shared provider, incarnation scope, atomic transaction builder, - region scanner, and completion router behind a feature flag. + region scanner, insertion-point completion gate, and completion router + behind a feature flag. 3. Run vNext relation completion in shadow mode while the legacy source remains visible. 4. Add column and namespace completion to the library and compare the golden diff --git a/src/vnext/codemirror/__tests__/sql-editor.test.ts b/src/vnext/codemirror/__tests__/sql-editor.test.ts index 69e41c7..9e4aa4b 100644 --- a/src/vnext/codemirror/__tests__/sql-editor.test.ts +++ b/src/vnext/codemirror/__tests__/sql-editor.test.ts @@ -1275,6 +1275,144 @@ describe("sqlEditor", () => { ); }); + it("denies SQL completion at an unmatched template EOF without removing external sources", async () => { + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const gate = vi.fn((state, position) => + !state.sliceDoc(0, position).endsWith("{") + ); + const support = sqlEditor({ + autocomplete: { + externalSources: [() => ({ + from: 15, + options: [{ label: "python_variable" }], + })], + isCompletionPositionAllowed: gate, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension, "SELECT * FROM {"); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + expect(harness.completeSignals).toHaveLength(0); + expect(currentCompletions(view.state)).toEqual([ + expect.objectContaining({ label: "python_variable" }), + ]); + expect(gate).toHaveBeenCalledWith(view.state, 15); + }); + + it("allows normal SQL completion through the position gate", async () => { + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const gate = vi.fn(() => true); + const support = sqlEditor({ + autocomplete: { + isCompletionPositionAllowed: gate, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + expect(harness.completeSignals).toHaveLength(1); + expect(currentCompletions(view.state)).toEqual([ + expect.objectContaining({ label: "users" }), + ]); + expect(gate).toHaveBeenCalledWith(view.state, 16); + }); + + it("fails a throwing position gate closed while retaining external sources", async () => { + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const support = sqlEditor({ + autocomplete: { + externalSources: [() => ({ + from: 15, + options: [{ label: "python_variable" }], + })], + isCompletionPositionAllowed: () => { + throw new Error("host state unavailable"); + }, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension, "SELECT * FROM {"); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + expect(harness.completeSignals).toHaveLength(0); + expect(currentCompletions(view.state)).toEqual([ + expect.objectContaining({ label: "python_variable" }), + ]); + }); + + it("cancels pending SQL completion when the position gate flips false", async () => { + let allowed = true; + const harness = fakeService( + () => new Promise(() => undefined), + ); + const support = sqlEditor({ + autocomplete: { + isCompletionPositionAllowed: () => allowed, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await vi.waitFor(() => { + expect(harness.completeSignals).toHaveLength(1); + }); + allowed = false; + view.dispatch({}); + await vi.waitFor(() => { + expect(harness.completeSignals[0]?.aborted).toBe(true); + }); + }); + + it("disposes rich info when the position gate flips false", async () => { + let allowed = true; + const destroys: Array> = []; + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const support = sqlEditor({ + autocomplete: { + infoResolver: () => { + const destroy = vi.fn(); + destroys.push(destroy); + return { + destroy, + dom: document.createElement("div"), + }; + }, + isCompletionPositionAllowed: () => allowed, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + await vi.waitFor(() => expect(destroys.length).toBeGreaterThan(0)); + const currentDestroy = destroys.at(-1); + if (!currentDestroy) throw new Error("Expected rich info"); + + allowed = false; + view.dispatch({}); + expect(currentDestroy).toHaveBeenCalledTimes(1); + }); + it.each([ "cancelled", "failed", diff --git a/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts new file mode 100644 index 0000000..3475b1f --- /dev/null +++ b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts @@ -0,0 +1,153 @@ +import { + currentCompletions, + startCompletion, +} from "@codemirror/autocomplete"; +import { EditorView } from "@codemirror/view"; +import { expect, test } from "vitest"; +import { + createSqlLanguageService, + duckdbDialect, +} from "../../index.js"; +import { sqlEditor } from "../index.js"; + +test("vNext completion gate routes unmatched template EOF to external sources", async () => { + const parent = document.createElement("div"); + document.body.append(parent); + let catalogSearches = 0; + const service = createSqlLanguageService({ + catalog: { + id: "browser-catalog", + search: async () => { + catalogSearches += 1; + return { + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "ready" }, + relations: [{ + canonicalPath: [ + { + quoted: false, + role: "schema", + value: "main", + }, + { + quoted: false, + role: "relation", + value: "users", + }, + ], + completionPathStart: 1, + entityId: "main.users", + matchQuality: "exact", + relationKind: "table", + }], + status: "ready" as const, + }; + }, + }, + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + autocomplete: { + externalSources: [(context) => ({ + from: context.pos, + options: [{ label: "python_variable" }], + })], + isCompletionPositionAllowed: (state, position) => { + const prefix = state.sliceDoc(0, position); + return prefix.lastIndexOf("{") <= prefix.lastIndexOf("}"); + }, + }, + initialContext: { + catalog: { + scope: "browser", + searchPath: [[{ quoted: false, value: "main" }]], + }, + dialect: "duckdb", + }, + initialEmbeddedRegions: [{ + from: 14, + language: "python", + to: 15, + }], + service, + }); + const view = new EditorView({ + doc: "SELECT * FROM {", + extensions: support.extension, + parent, + selection: { anchor: 15 }, + }); + + expect(startCompletion(view)).toBe(true); + await expect.poll(() => + currentCompletions(view.state).map((item) => item.label) + ).toEqual(["python_variable"]); + expect(catalogSearches).toBe(0); + + view.destroy(); + service.dispose(); + parent.remove(); +}); + +test("vNext completion gate permits normal SQL in a browser editor", async () => { + const parent = document.createElement("div"); + document.body.append(parent); + const service = createSqlLanguageService({ + catalog: { + id: "browser-catalog", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "ready" }, + relations: [{ + canonicalPath: [ + { + quoted: false, + role: "schema", + value: "main", + }, + { + quoted: false, + role: "relation", + value: "users", + }, + ], + completionPathStart: 1, + entityId: "main.users", + matchQuality: "exact", + relationKind: "table", + }], + status: "ready" as const, + }), + }, + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + autocomplete: { + isCompletionPositionAllowed: () => true, + }, + initialContext: { + catalog: { + scope: "browser", + searchPath: [[{ quoted: false, value: "main" }]], + }, + dialect: "duckdb", + }, + service, + }); + const documentText = "SELECT * FROM us"; + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + selection: { anchor: documentText.length }, + }); + + expect(startCompletion(view)).toBe(true); + await expect.poll(() => + currentCompletions(view.state).map((item) => item.label) + ).toContain("users"); + + view.destroy(); + service.dispose(); + parent.remove(); +}); diff --git a/src/vnext/codemirror/sql-editor.ts b/src/vnext/codemirror/sql-editor.ts index 54a0cd6..887b458 100644 --- a/src/vnext/codemirror/sql-editor.ts +++ b/src/vnext/codemirror/sql-editor.ts @@ -15,6 +15,7 @@ import { Prec, StateEffect, StateField, + type EditorState, type EditorSelection, type Extension, type StateEffectType, @@ -63,6 +64,10 @@ export interface SqlEditorAutocompleteOptions { readonly defaultKeymap?: boolean; readonly externalSources?: readonly CompletionSource[]; readonly infoResolver?: SqlCompletionInfoResolver; + readonly isCompletionPositionAllowed?: ( + state: EditorState, + position: number, + ) => boolean; readonly maxRenderedOptions?: number; readonly selectOnOpen?: boolean; readonly updateSyncTime?: number; @@ -198,6 +203,7 @@ function haveOneEditRange(items: readonly SqlCompletionItem[]): boolean { } function completionType(item: SqlCompletionItem): string { + if (item.kind === "column") return "property"; return item.relationKind === "cte" ? "type" : "table"; } @@ -260,6 +266,7 @@ export function createSqlEditorInternal< const { externalSources = [], infoResolver, + isCompletionPositionAllowed, ...autocompleteOptions } = autocomplete; @@ -379,6 +386,17 @@ export function createSqlEditorInternal< ); } + #completionPositionIsAllowed( + state: EditorState, + position: number, + ): boolean { + try { + return isCompletionPositionAllowed?.(state, position) ?? true; + } catch { + return false; + } + } + #scheduleClose(): void { const sequence = this.#sequence; runtime.queueMicrotask(() => { @@ -539,6 +557,14 @@ export function createSqlEditorInternal< context: CompletionContext, ): Promise => { this.#clearCompletionState(); + if ( + !this.#completionPositionIsAllowed( + context.state, + context.pos, + ) + ) { + return null; + } const capture: CompletionCapture = { contextGeneration: this.#contextGeneration, document: this.#view.state.doc, @@ -586,6 +612,17 @@ export function createSqlEditorInternal< } return null; } + if ( + !this.#completionPositionIsAllowed( + this.#view.state, + context.pos, + ) + ) { + if (this.#active === active) { + this.#clearCompletionState(); + } + return null; + } if ( controller.signal.aborted || this.#active !== active || @@ -651,6 +688,14 @@ export function createSqlEditorInternal< }; readonly update = (update: ViewUpdate): void => { + if ( + !this.#completionPositionIsAllowed( + update.state, + update.state.selection.main.head, + ) + ) { + this.#clearCompletionState(); + } let contextChanged = false; let regionsChanged = false; for (const transaction of update.transactions) { diff --git a/test/vnext-types/marimo-sql-migration.test-d.ts b/test/vnext-types/marimo-sql-migration.test-d.ts index ab2d76b..b116f3a 100644 --- a/test/vnext-types/marimo-sql-migration.test-d.ts +++ b/test/vnext-types/marimo-sql-migration.test-d.ts @@ -3,6 +3,7 @@ import type { } from "@codemirror/autocomplete"; import type { ChangeSpec, + EditorState, StateEffectType, TransactionSpec, } from "@codemirror/state"; @@ -228,6 +229,38 @@ function pythonTemplateRegions( return regions; } +/** + * Embedded regions are half-open, so an unmatched expression ending at EOF + * also needs an insertion-point gate at `doc.length`. + */ +function sqlCompletionPositionAllowed( + state: EditorState, + position: number, +): boolean { + const prefix = state.sliceDoc(0, position); + let inPython = false; + for (let index = 0; index < prefix.length; index += 1) { + if (inPython) { + if (prefix[index] === "}") inPython = false; + continue; + } + if ( + prefix[index] === "{" && + prefix[index + 1] !== "{" + ) { + inPython = true; + continue; + } + if ( + prefix[index] === "{" && + prefix[index + 1] === "{" + ) { + index += 1; + } + } + return !inPython; +} + interface VnextCompletionRoute { readonly connection: MarimoConnection; readonly dialect: VnextDialectId; @@ -274,6 +307,7 @@ function createVnextSupport( keywordCompletionSource, ], infoResolver, + isCompletionPositionAllowed: sqlCompletionPositionAllowed, }, initialContext: contextFor(route), initialEmbeddedRegions: pythonTemplateRegions(initialText), From 995182c31120ba88d2e572ed7e22c9df1baf4fd5 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:40:04 +0800 Subject: [PATCH 05/25] feat(vnext): add batched column catalog authority --- docs/vnext/column-catalog-authority.md | 57 ++ .../column-catalog-batch-coordinator.bench.ts | 101 +++ .../column-catalog-batch-coordinator.test.ts | 408 ++++++++++ .../__tests__/column-catalog-boundary.test.ts | 472 ++++++++++++ src/vnext/column-catalog-batch-coordinator.ts | 570 ++++++++++++++ src/vnext/column-catalog-boundary.ts | 712 ++++++++++++++++++ src/vnext/column-catalog-types.ts | 82 ++ 7 files changed, 2402 insertions(+) create mode 100644 docs/vnext/column-catalog-authority.md create mode 100644 src/vnext/__tests__/column-catalog-batch-coordinator.bench.ts create mode 100644 src/vnext/__tests__/column-catalog-batch-coordinator.test.ts create mode 100644 src/vnext/__tests__/column-catalog-boundary.test.ts create mode 100644 src/vnext/column-catalog-batch-coordinator.ts create mode 100644 src/vnext/column-catalog-boundary.ts create mode 100644 src/vnext/column-catalog-types.ts diff --git a/docs/vnext/column-catalog-authority.md b/docs/vnext/column-catalog-authority.md new file mode 100644 index 0000000..5216f7a --- /dev/null +++ b/docs/vnext/column-catalog-authority.md @@ -0,0 +1,57 @@ +# vNext Column Catalog Authority + +Status: internal vertical-slice contract + +Column discovery is lazy, provider-owned, and batched. A completion request +sends every unresolved relation reference in one provider invocation. Each +reference carries a caller-local `requestKey`, a decoded identifier path, and +an optional previously authenticated relation entity ID. The provider resolves +paths against the supplied catalog scope, search paths, and dialect. + +The provider returns stable relation and column entity IDs. Every accepted +column contains: + +- a canonical `SqlIdentifierComponent`; +- bounded provider-rendered `insertText`; +- a stable column entity ID and ordinal; and +- immutable provenance containing provider, scope, epoch, relation, and column + identities. + +Responses describe each requested relation independently as: + +- `ready` with complete or partial column coverage; +- `loading`; or +- `failed` with a normalized code and retry policy. + +Missing, extra, conflicting, oversized, accessor-backed, or malformed data is +rejected at the provider boundary. Relations and columns have deterministic +code-unit order. Duplicate request keys and conflicting stable IDs fail closed; +identical duplicate columns are deduplicated. + +## Epoch and cache behavior + +Cold requests use `expectedEpoch: null`. A response always supplies an epoch. +An owner remembers that observation, and later null-epoch requests reuse only +cache entries from the observed epoch. Explicit expected epochs never reuse +entries from another epoch. The cache is LRU-bounded by relation entries. +Only complete ready results are cached. Partial, loading, and failed results +remain visible to the caller but are eligible for another provider request. + +The initial coordinator does not subscribe to catalog invalidations. Until it +is connected to the relation catalog's private epoch coordinator, a host must +supersede or dispose the column owner when catalog authority changes. Session +integration must not treat a cached complete result as current across a known +catalog revision. + +## Lifecycle + +One owner represents one document/session authority for a scope and dialect. +Starting a newer request supersedes and aborts that owner's prior request. +Explicit cancellation settles as cancelled. Owner or coordinator disposal +aborts outstanding work and settles it as unavailable/disposed. Provider +rejections, throws, and malformed responses are contained; late settlements +cannot publish or populate the cache. + +The coordinator batches a request once, never once per relation. Cache hits and +misses are composed deterministically while only misses are sent to the +provider. diff --git a/src/vnext/__tests__/column-catalog-batch-coordinator.bench.ts b/src/vnext/__tests__/column-catalog-batch-coordinator.bench.ts new file mode 100644 index 0000000..e585907 --- /dev/null +++ b/src/vnext/__tests__/column-catalog-batch-coordinator.bench.ts @@ -0,0 +1,101 @@ +import { bench, describe } from "vitest"; +import { + createSqlColumnCatalogBatchCoordinator, +} from "../column-catalog-batch-coordinator.js"; +import type { + SqlColumnCatalogBatchRequest, +} from "../column-catalog-types.js"; + +const epoch = Object.freeze({ generation: 1, token: "bench" }); +const relations = Object.freeze( + Array.from({ length: 64 }, (_, index) => + Object.freeze({ + path: Object.freeze([ + Object.freeze({ + quoted: false, + value: `relation_${index}`, + }), + ]), + requestKey: `relation_${index}`, + }) + ), +); +const searchPaths = Object.freeze([ + Object.freeze([ + Object.freeze({ quoted: false, value: "main" }), + ]), +]); + +function response(request: SqlColumnCatalogBatchRequest) { + return Object.freeze({ + epoch, + relations: Object.freeze(request.relations.map((relation) => + Object.freeze({ + columns: Object.freeze( + Array.from({ length: 64 }, (_, ordinal) => + Object.freeze({ + columnEntityId: + `${relation.requestKey}:column:${ordinal}`, + identifier: Object.freeze({ + quoted: false, + value: `column_${ordinal}`, + }), + insertText: `column_${ordinal}`, + ordinal, + }) + ), + ), + coverage: "complete", + relationEntityId: `entity:${relation.requestKey}`, + requestKey: relation.requestKey, + status: "ready", + }) + )), + }); +} + +function owner() { + const created = createSqlColumnCatalogBatchCoordinator({ + maxCacheEntries: 128, + provider: { + id: "benchmark", + loadColumns: (request: SqlColumnCatalogBatchRequest) => + Promise.resolve(response(request)), + }, + }); + if (created.status !== "created") throw new Error("coordinator"); + const prepared = created.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "benchmark", + }); + if (prepared.status !== "prepared") throw new Error("owner"); + return prepared.owner; +} + +describe("column catalog batch coordinator", () => { + bench("cold 64 relation x 64 column batch", async () => { + const current = owner(); + await current.request({ + expectedEpoch: null, + relations, + searchPaths, + }).result; + current.dispose(); + }); + + const warmOwner = owner(); + const warmReady = warmOwner.request({ + expectedEpoch: epoch, + relations, + searchPaths, + }).result; + + bench("warm 64 relation cache projection", async () => { + await warmReady; + await warmOwner.request({ + expectedEpoch: epoch, + relations, + searchPaths, + }).result; + }); +}); diff --git a/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts new file mode 100644 index 0000000..174f0cc --- /dev/null +++ b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts @@ -0,0 +1,408 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createSqlColumnCatalogBatchCoordinator, +} from "../column-catalog-batch-coordinator.js"; +import type { + SqlColumnCatalogBatchRequest, +} from "../column-catalog-types.js"; +import type { SqlCatalogEpoch } from "../relation-completion-types.js"; + +const epoch: SqlCatalogEpoch = + Object.freeze({ generation: 1, token: "one" }); +const nextEpoch: SqlCatalogEpoch = + Object.freeze({ generation: 2, token: "two" }); + +function reference(requestKey: string, name = requestKey) { + return Object.freeze({ + path: Object.freeze([ + Object.freeze({ quoted: false, value: name }), + ]), + requestKey, + }); +} + +function ready( + request: SqlColumnCatalogBatchRequest, + suffix = "", +) { + return { + epoch: request.expectedEpoch ?? epoch, + relations: request.relations.map((relation) => ({ + columns: [{ + columnEntityId: `column-${relation.requestKey}${suffix}`, + identifier: { + quoted: false, + value: `value_${relation.requestKey}${suffix}`, + }, + insertText: `value_${relation.requestKey}${suffix}`, + ordinal: 0, + }], + coverage: "complete", + relationEntityId: + relation.relationEntityId ?? `relation-${relation.requestKey}`, + requestKey: relation.requestKey, + status: "ready", + })), + }; +} + +function setup( + loadColumns: ( + request: SqlColumnCatalogBatchRequest, + signal: AbortSignal, + ) => unknown, + maxCacheEntries = 32, +) { + const result = createSqlColumnCatalogBatchCoordinator({ + maxCacheEntries, + provider: { id: "catalog", loadColumns }, + }); + if (result.status !== "created") { + throw new Error("Expected coordinator"); + } + const prepared = result.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + }); + if (prepared.status !== "prepared") { + throw new Error("Expected owner"); + } + return { + coordinator: result.coordinator, + owner: prepared.owner, + }; +} + +function input( + relations = [reference("users"), reference("events")], + epoch_: SqlCatalogEpoch | null = epoch, +) { + return { + expectedEpoch: epoch_, + relations, + searchPaths: [[{ quoted: false, value: "main" }]], + }; +} + +function deferred() { + let resolve: (value: Value) => void = (): void => {}; + let reject: (reason?: unknown) => void = (): void => {}; + const promise = new Promise((resolve_, reject_) => { + resolve = resolve_; + reject = reject_; + }); + return { promise, reject, resolve }; +} + +describe("column catalog batch coordinator", () => { + it("loads every missing relation in one deterministic batch", async () => { + const requests: SqlColumnCatalogBatchRequest[] = []; + const { coordinator, owner } = setup((request) => { + requests.push(request); + return ready(request); + }); + const outcome = await owner.request(input([ + reference("users"), + reference("events"), + reference("accounts"), + ])).result; + + expect(requests).toHaveLength(1); + expect(requests[0]?.relations.map((relation) => + relation.requestKey + )).toEqual(["accounts", "events", "users"]); + expect(outcome).toMatchObject({ + providerId: "catalog", + relations: [ + { requestKey: "accounts", status: "ready" }, + { requestKey: "events", status: "ready" }, + { requestKey: "users", status: "ready" }, + ], + status: "usable", + }); + expect(Object.isFrozen(outcome)).toBe(true); + if (outcome.status === "usable") { + expect(Object.isFrozen(outcome.relations)).toBe(true); + } + coordinator.dispose(); + }); + + it("caches ready relations by authority identity and remaps request keys", async () => { + const requests: SqlColumnCatalogBatchRequest[] = []; + const { owner } = setup((request) => { + requests.push(request); + return ready(request); + }); + await owner.request(input([reference("first", "users")])).result; + const cached = await owner.request(input([ + reference("renamed", "users"), + ])).result; + await owner.request(input( + [reference("new-epoch", "users")], + nextEpoch, + )).result; + + expect(requests).toHaveLength(2); + expect(cached).toMatchObject({ + relations: [{ + relationEntityId: "relation-first", + requestKey: "renamed", + }], + status: "usable", + }); + }); + + it("supports cold null epochs and reuses the observed response epoch", async () => { + const expected: Array = []; + const { owner } = setup((request) => { + expected.push(request.expectedEpoch); + return { + ...ready(request), + epoch, + }; + }); + await owner.request(input([reference("users")], null)).result; + const cached = await owner.request( + input([reference("renamed", "users")], null), + ).result; + + expect(expected).toEqual([null]); + expect(cached).toMatchObject({ + relations: [{ requestKey: "renamed" }], + status: "usable", + }); + }); + + it("combines cached and loaded relations without changing order", async () => { + const requests: SqlColumnCatalogBatchRequest[] = []; + const { owner } = setup((request) => { + requests.push(request); + return ready(request); + }); + await owner.request(input([reference("users")])).result; + const mixed = await owner.request(input([ + reference("users"), + reference("events"), + ])).result; + + expect(requests).toHaveLength(2); + expect(requests[1]?.relations.map((relation) => + relation.requestKey + )).toEqual(["events"]); + expect(mixed).toMatchObject({ + relations: [ + { requestKey: "events" }, + { requestKey: "users" }, + ], + status: "usable", + }); + }); + + it("caches only complete ready coverage", async () => { + let calls = 0; + const { owner } = setup((request) => { + calls += 1; + return { + epoch: request.expectedEpoch ?? epoch, + relations: request.relations.map((relation, index) => + index === 0 + ? { requestKey: relation.requestKey, status: "loading" } + : index === 1 + ? { + columns: [], + coverage: "partial", + relationEntityId: `relation-${relation.requestKey}`, + requestKey: relation.requestKey, + status: "ready", + } + : { + code: "unavailable", + requestKey: relation.requestKey, + retry: "next-request", + status: "failed", + } + ), + }; + }); + const uncached = [ + reference("loading"), + reference("partial"), + reference("failed"), + ]; + const first = await owner.request(input(uncached)).result; + const second = await owner.request(input(uncached)).result; + + expect(first).toMatchObject({ + relations: [ + { requestKey: "failed", status: "loading" }, + { requestKey: "loading", status: "ready" }, + { requestKey: "partial", status: "failed" }, + ], + status: "usable", + }); + expect(second.status).toBe("usable"); + expect(calls).toBe(2); + }); + + it("aborts and settles cancelled work exactly once", async () => { + const work = deferred(); + const signals: AbortSignal[] = []; + const { owner } = setup((_request, signal_) => { + signals.push(signal_); + return work.promise; + }); + const ticket = owner.request(input()); + ticket.cancel(); + ticket.cancel(); + + await expect(ticket.result).resolves.toEqual({ status: "cancelled" }); + expect(signals[0]?.aborted).toBe(true); + work.resolve({}); + await Promise.resolve(); + await expect(ticket.result).resolves.toEqual({ status: "cancelled" }); + }); + + it("supersedes prior owner work but isolates other owners", async () => { + const pending: Array>> = []; + const { coordinator, owner } = setup(() => { + const work = deferred(); + pending.push(work); + return work.promise; + }); + const otherResult = coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + }); + if (otherResult.status !== "prepared") { + throw new Error("Expected second owner"); + } + const first = owner.request(input()); + const other = otherResult.owner.request(input()); + const second = owner.request(input([reference("next")])); + + await expect(first.result).resolves.toEqual({ + status: "superseded", + }); + let otherSettled = false; + void other.result.then(() => { + otherSettled = true; + }); + await Promise.resolve(); + expect(otherSettled).toBe(false); + second.cancel(); + other.cancel(); + }); + + it("disposes owners and coordinator with prompt aborts", async () => { + const signals: AbortSignal[] = []; + const { coordinator, owner } = setup((_request, signal) => { + signals.push(signal); + return new Promise(() => undefined); + }); + const ownerTicket = owner.request(input()); + owner.dispose(); + await expect(ownerTicket.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + expect(signals[0]?.aborted).toBe(true); + await expect(owner.request(input()).result).resolves.toMatchObject({ + reason: "disposed", + }); + + const next = coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + }); + if (next.status !== "prepared") throw new Error("Expected owner"); + const coordinatorTicket = next.owner.request(input()); + coordinator.dispose(); + coordinator.dispose(); + await expect(coordinatorTicket.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + expect(coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + })).toEqual({ reason: "disposed", status: "unavailable" }); + }); + + it.each([ + { + expected: "provider-failed", + load: () => { + throw new Error("sync"); + }, + }, + { + expected: "provider-failed", + load: () => Promise.reject(new Error("async")), + }, + { + expected: "malformed-response", + load: () => ({ epoch, relations: [] }), + }, + ])("contains provider failure as $expected", async ({ expected, load }) => { + const { owner } = setup(load); + await expect(owner.request(input()).result).resolves.toEqual({ + reason: expected, + status: "unavailable", + }); + }); + + it("bounds cache entries with deterministic LRU eviction", async () => { + const calls = new Map(); + const { owner } = setup((request) => { + const key = request.relations[0]?.path[0]?.value ?? ""; + calls.set(key, (calls.get(key) ?? 0) + 1); + return ready(request); + }, 2); + await owner.request(input([reference("a")])).result; + await owner.request(input([reference("b")])).result; + await owner.request(input([reference("a")])).result; + await owner.request(input([reference("c")])).result; + await owner.request(input([reference("b")])).result; + + expect(calls).toEqual(new Map([ + ["a", 1], + ["b", 2], + ["c", 1], + ])); + }); + + it("rejects invalid providers, options, owners, and requests", async () => { + expect(createSqlColumnCatalogBatchCoordinator(null)).toEqual({ + reason: "invalid-provider", + status: "unavailable", + }); + expect(createSqlColumnCatalogBatchCoordinator({ + maxCacheEntries: 0, + provider: { id: "catalog", loadColumns: vi.fn() }, + })).toEqual({ + reason: "invalid-options", + status: "unavailable", + }); + const created = createSqlColumnCatalogBatchCoordinator({ + provider: { id: "catalog", loadColumns: vi.fn() }, + }); + if (created.status !== "created") throw new Error("Expected coordinator"); + expect(created.coordinator.prepareOwner({ + dialectId: "", + scope: "scope", + })).toMatchObject({ status: "unavailable" }); + const prepared = created.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + }); + if (prepared.status !== "prepared") throw new Error("Expected owner"); + await expect(prepared.owner.request({ + expectedEpoch: epoch, + relations: [], + searchPaths: [], + }).result).resolves.toEqual({ + reason: "invalid-request", + status: "unavailable", + }); + }); +}); diff --git a/src/vnext/__tests__/column-catalog-boundary.test.ts b/src/vnext/__tests__/column-catalog-boundary.test.ts new file mode 100644 index 0000000..b067603 --- /dev/null +++ b/src/vnext/__tests__/column-catalog-boundary.test.ts @@ -0,0 +1,472 @@ +import { describe, expect, it, vi } from "vitest"; +import { + captureSqlColumnCatalogProvider, + createSqlColumnCatalogBatchRequest, + decodeSqlColumnCatalogBatchResponse, + MAX_COLUMN_BATCH_RELATIONS, + MAX_COLUMN_ENTITY_ID_LENGTH, + MAX_COLUMNS_PER_RELATION, + resolveSqlColumnCatalogProvider, +} from "../column-catalog-boundary.js"; + +const epoch = Object.freeze({ generation: 7, token: "epoch-7" }); +const path = Object.freeze([ + Object.freeze({ quoted: false, value: "users" }), +]); + +function request() { + const result = createSqlColumnCatalogBatchRequest({ + dialectId: "duckdb", + expectedEpoch: epoch, + relations: [ + { path, requestKey: "users" }, + { + path: [{ quoted: false, value: "events" }], + relationEntityId: "relation-events", + requestKey: "events", + }, + ], + scope: "connection-1", + searchPaths: [[{ quoted: false, value: "main" }]], + }); + if (result.status !== "accepted") { + throw new Error("Expected valid request"); + } + return result.value; +} + +function provider(loadColumns: (...arguments_: unknown[]) => unknown) { + const captured = captureSqlColumnCatalogProvider({ + id: "catalog", + loadColumns, + }); + if (captured.status !== "accepted") { + throw new Error("Expected valid provider"); + } + return captured.value; +} + +function readyResponse() { + return { + epoch, + relations: [ + { + columns: [ + { + columnEntityId: "column-name", + dataType: "VARCHAR", + identifier: { quoted: false, value: "name" }, + insertText: "name", + ordinal: 1, + }, + { + columnEntityId: "column-id", + detail: "primary key", + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: 0, + }, + ], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + { + code: "unavailable", + requestKey: "events", + retry: "next-request", + status: "failed", + }, + ], + }; +} + +describe("column catalog provider boundary", () => { + it("captures methods without retaining provider receivers", () => { + const calls: unknown[][] = []; + const captured = provider(function ( + this: unknown, + ...arguments_: unknown[] + ) { + calls.push([this, ...arguments_]); + return null; + }); + const resolved = resolveSqlColumnCatalogProvider(captured); + const signal = new AbortController().signal; + resolved?.loadColumns(request(), signal); + + expect(resolved?.id).toBe("catalog"); + expect(calls).toEqual([[undefined, request(), signal]]); + expect(Object.isFrozen(resolved)).toBe(true); + expect(resolveSqlColumnCatalogProvider({})).toBeNull(); + }); + + it("rejects inherited, accessor, and oversized provider data", () => { + let calls = 0; + const accessor = Object.defineProperty( + { id: "catalog" }, + "loadColumns", + { + get: () => { + calls += 1; + return () => undefined; + }, + }, + ); + expect(captureSqlColumnCatalogProvider(accessor)).toMatchObject({ + reason: "invalid-shape", + status: "malformed", + }); + expect(calls).toBe(0); + expect(captureSqlColumnCatalogProvider( + Object.create({ id: "catalog", loadColumns: () => undefined }), + ).status).toBe("malformed"); + expect(captureSqlColumnCatalogProvider({ + id: "x".repeat(257), + loadColumns: () => undefined, + }).status).toBe("malformed"); + expect(captureSqlColumnCatalogProvider(null).status).toBe("malformed"); + }); +}); + +describe("column catalog batch request boundary", () => { + it("normalizes paths, search paths, and deterministic request order", () => { + const result = createSqlColumnCatalogBatchRequest({ + dialectId: "duckdb", + expectedEpoch: epoch, + relations: [ + { path, requestKey: "z" }, + { path, relationEntityId: "stable-a", requestKey: "a" }, + ], + scope: "scope", + searchPaths: [[{ quoted: true, value: "Main" }]], + }); + expect(result).toMatchObject({ + status: "accepted", + value: { + relations: [ + { relationEntityId: "stable-a", requestKey: "a" }, + { requestKey: "z" }, + ], + }, + }); + if (result.status === "accepted") { + expect(Object.isFrozen(result.value)).toBe(true); + expect(Object.isFrozen(result.value.relations)).toBe(true); + expect(Object.isFrozen(result.value.relations[0]?.path)).toBe(true); + expect(Object.isFrozen(result.value.searchPaths)).toBe(true); + } + }); + + it("rejects malformed, duplicate, sparse, and resource-heavy requests", () => { + const base = { + dialectId: "duckdb", + expectedEpoch: epoch, + scope: "scope", + searchPaths: [], + }; + for (const relations of [ + [], + [{ path: [], requestKey: "empty-path" }], + [{ path, requestKey: "" }], + [ + { path, requestKey: "same" }, + { path, requestKey: "same" }, + ], + Array.from( + { length: MAX_COLUMN_BATCH_RELATIONS + 1 }, + (_, index) => ({ path, requestKey: `r${index}` }), + ), + ]) { + expect(createSqlColumnCatalogBatchRequest({ + ...base, + relations, + }).status).toBe("malformed"); + } + const sparse: unknown[] = []; + sparse.length = 1; + expect(createSqlColumnCatalogBatchRequest({ + ...base, + relations: sparse, + }).status).toBe("malformed"); + expect(createSqlColumnCatalogBatchRequest({ + ...base, + relations: [{ + path, + relationEntityId: "x".repeat(MAX_COLUMN_ENTITY_ID_LENGTH + 1), + requestKey: "long", + }], + }).status).toBe("malformed"); + expect(createSqlColumnCatalogBatchRequest( + Object.create(base), + ).status).toBe("malformed"); + }); + + it("does not invoke request accessors or hostile array iterators", () => { + let getterCalls = 0; + const input = Object.defineProperty( + { + dialectId: "duckdb", + epoch, + relations: [{ path, requestKey: "users" }], + searchPaths: [], + }, + "scope", + { + get: () => { + getterCalls += 1; + return "scope"; + }, + }, + ); + expect(createSqlColumnCatalogBatchRequest(input).status).toBe( + "malformed", + ); + expect(getterCalls).toBe(0); + + const relations = [{ path, requestKey: "users" }]; + Object.defineProperty(relations, Symbol.iterator, { + value: () => { + throw new Error("iterator must not run"); + }, + }); + expect(createSqlColumnCatalogBatchRequest({ + dialectId: "duckdb", + expectedEpoch: epoch, + relations, + scope: "scope", + searchPaths: [], + }).status).toBe("accepted"); + }); +}); + +describe("column catalog response boundary", () => { + it("decodes all coverage states with stable frozen provenance", () => { + const captured = provider(() => undefined); + const result = decodeSqlColumnCatalogBatchResponse( + captured, + request(), + readyResponse(), + ); + expect(result).toMatchObject({ + status: "accepted", + value: { + relations: [ + { + code: "unavailable", + requestKey: "events", + status: "failed", + }, + { + columns: [ + { columnEntityId: "column-id", ordinal: 0 }, + { columnEntityId: "column-name", ordinal: 1 }, + ], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + ], + }, + }); + if (result.status === "accepted") { + const users = result.value.relations[1]; + expect(Object.isFrozen(result.value)).toBe(true); + expect(Object.isFrozen(result.value.relations)).toBe(true); + if (users?.status === "ready") { + expect(users.columns[0]?.provenance).toEqual({ + columnEntityId: "column-id", + epoch, + providerId: "catalog", + relationEntityId: "relation-users", + scope: "connection-1", + }); + expect(Object.isFrozen(users.columns[0]?.provenance)).toBe(true); + } + } + }); + + it("accepts loading, partial, and identical duplicate columns", () => { + const captured = provider(() => undefined); + const request_ = request(); + const column = { + columnEntityId: "same", + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: 0, + }; + const result = decodeSqlColumnCatalogBatchResponse( + captured, + request_, + { + epoch, + relations: [ + { + columns: [column, { ...column }], + coverage: "partial", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + { requestKey: "events", status: "loading" }, + ], + }, + ); + expect(result).toMatchObject({ + status: "accepted", + value: { + relations: [ + { requestKey: "events", status: "loading" }, + { + columns: [{ columnEntityId: "same" }], + coverage: "partial", + requestKey: "users", + }, + ], + }, + }); + }); + + it.each([ + { + name: "wrong epoch", + value: { ...readyResponse(), epoch: { generation: 8, token: "x" } }, + }, + { + name: "missing relation", + value: { epoch, relations: [readyResponse().relations[0]] }, + }, + { + name: "unexpected relation", + value: { + epoch, + relations: [ + ...readyResponse().relations, + { requestKey: "other", status: "loading" }, + ], + }, + }, + { + name: "duplicate request key", + value: { + epoch, + relations: [ + readyResponse().relations[0], + readyResponse().relations[0], + ], + }, + }, + { + name: "conflicting known entity", + value: { + epoch, + relations: [ + readyResponse().relations[0], + { + columns: [], + coverage: "complete", + relationEntityId: "wrong-events-id", + requestKey: "events", + status: "ready", + }, + ], + }, + }, + { + name: "conflicting duplicate column", + value: { + epoch, + relations: [ + { + ...readyResponse().relations[0], + columns: [ + { + columnEntityId: "x", + identifier: { quoted: false, value: "a" }, + insertText: "a", + ordinal: 0, + }, + { + columnEntityId: "x", + identifier: { quoted: false, value: "b" }, + insertText: "b", + ordinal: 0, + }, + ], + }, + readyResponse().relations[1], + ], + }, + }, + { + name: "too many columns", + value: { + epoch, + relations: [ + { + ...readyResponse().relations[0], + columns: Array.from( + { length: MAX_COLUMNS_PER_RELATION + 1 }, + (_, ordinal) => ({ + columnEntityId: `c${ordinal}`, + identifier: { + quoted: false, + value: `c${ordinal}`, + }, + insertText: `c${ordinal}`, + ordinal, + }), + ), + }, + readyResponse().relations[1], + ], + }, + }, + ])("rejects $name", ({ value }) => { + expect(decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + value, + ).status).toBe("malformed"); + }); + + it("contains response accessors, iterators, and invalid provider handles", () => { + let getterCalls = 0; + const response = Object.defineProperty( + { epoch }, + "relations", + { + get: () => { + getterCalls += 1; + return []; + }, + }, + ); + expect(decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + response, + ).status).toBe("malformed"); + expect(getterCalls).toBe(0); + + const value = readyResponse(); + Object.defineProperty(value.relations, Symbol.iterator, { + value: vi.fn(() => { + throw new Error("iterator must not run"); + }), + }); + expect(decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + value, + ).status).toBe("accepted"); + const invalid = Reflect.apply( + decodeSqlColumnCatalogBatchResponse, + undefined, + [{}, request(), value], + ); + expect(invalid.status).toBe("malformed"); + }); +}); diff --git a/src/vnext/column-catalog-batch-coordinator.ts b/src/vnext/column-catalog-batch-coordinator.ts new file mode 100644 index 0000000..e00b071 --- /dev/null +++ b/src/vnext/column-catalog-batch-coordinator.ts @@ -0,0 +1,570 @@ +import { + captureSqlColumnCatalogProvider, + createSqlColumnCatalogBatchRequest, + decodeSqlColumnCatalogBatchResponse, + MAX_COLUMN_DIALECT_ID_LENGTH, + MAX_COLUMN_SCOPE_LENGTH, + resolveSqlColumnCatalogProvider, + type CapturedSqlColumnCatalogProvider, + type SqlCapturedColumnCatalogProviderContext, +} from "./column-catalog-boundary.js"; +import type { + SqlColumnCatalogBatchRequest, + SqlColumnCatalogRelationReference, + SqlColumnCatalogRelationResult, +} from "./column-catalog-types.js"; +import type { + SqlCatalogEpoch, +} from "./relation-completion-types.js"; +import type { SqlIdentifierPath } from "./types.js"; + +export const DEFAULT_COLUMN_CACHE_ENTRIES = 1_024; +export const MAX_COLUMN_CACHE_ENTRIES = 8_192; + +export interface SqlColumnCatalogOwnerOptions { + readonly dialectId: string; + readonly scope: string; +} + +export interface SqlColumnCatalogBatchInput { + readonly expectedEpoch: SqlCatalogEpoch | null; + readonly relations: readonly SqlColumnCatalogRelationReference[]; + readonly searchPaths: readonly SqlIdentifierPath[]; +} + +export type SqlColumnCatalogBatchOutcome = + | { + readonly epoch: SqlCatalogEpoch; + readonly providerId: string; + readonly relations: readonly SqlColumnCatalogRelationResult[]; + readonly scope: string; + readonly status: "usable"; + } + | { + readonly status: "cancelled"; + } + | { + readonly status: "superseded"; + } + | { + readonly reason: + | "disposed" + | "invalid-request" + | "malformed-response" + | "provider-failed"; + readonly status: "unavailable"; + }; + +export interface SqlColumnCatalogBatchTicket { + readonly cancel: (this: void) => void; + readonly result: Promise; +} + +export interface SqlColumnCatalogBatchOwner { + readonly dispose: (this: void) => void; + readonly request: ( + this: void, + input: SqlColumnCatalogBatchInput, + ) => SqlColumnCatalogBatchTicket; +} + +export type SqlColumnCatalogBatchOwnerResult = + | { + readonly owner: SqlColumnCatalogBatchOwner; + readonly status: "prepared"; + } + | { + readonly reason: "disposed" | "invalid-options"; + readonly status: "unavailable"; + }; + +export interface SqlColumnCatalogBatchCoordinator { + readonly dispose: (this: void) => void; + readonly prepareOwner: ( + this: void, + options: SqlColumnCatalogOwnerOptions, + ) => SqlColumnCatalogBatchOwnerResult; + readonly providerId: string; +} + +export type SqlColumnCatalogBatchCoordinatorResult = + | { + readonly coordinator: SqlColumnCatalogBatchCoordinator; + readonly status: "created"; + } + | { + readonly reason: "invalid-options" | "invalid-provider"; + readonly status: "unavailable"; + }; + +interface CoordinatorState { + readonly cache: Map; + readonly capturedProvider: CapturedSqlColumnCatalogProvider; + readonly context: SqlCapturedColumnCatalogProviderContext; + disposed: boolean; + readonly maxCacheEntries: number; + readonly owners: Set; +} + +interface OwnerState { + active: ConsumerState | null; + readonly dialectId: string; + disposed: boolean; + owner: CoordinatorState | null; + observedEpoch: SqlCatalogEpoch | null; + readonly scope: string; +} + +interface ConsumerState { + readonly controller: AbortController; + readonly owner: OwnerState; + resolve: ((value: SqlColumnCatalogBatchOutcome) => void) | null; + settled: boolean; +} + +type ReadyRelation = Extract< + SqlColumnCatalogRelationResult, + { readonly status: "ready" } +>; + +const CANCELLED: SqlColumnCatalogBatchOutcome = + Object.freeze({ status: "cancelled" }); +const SUPERSEDED: SqlColumnCatalogBatchOutcome = + Object.freeze({ status: "superseded" }); + +function unavailable( + reason: Extract< + SqlColumnCatalogBatchOutcome, + { readonly status: "unavailable" } + >["reason"], +): SqlColumnCatalogBatchOutcome { + return Object.freeze({ reason, status: "unavailable" }); +} + +function dataProperty( + value: unknown, + key: string, +): { readonly found: true; readonly value: unknown } | null { + if (value === null || typeof value !== "object") return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, key); + } catch { + return null; + } + return descriptor && "value" in descriptor + ? { found: true, value: descriptor.value } + : null; +} + +function boundedString(value: unknown, maximum: number): string | null { + return typeof value === "string" && + value.length > 0 && + value.length <= maximum + ? value + : null; +} + +function cacheKey( + request: SqlColumnCatalogBatchRequest, + reference: SqlColumnCatalogRelationReference, + epoch: SqlCatalogEpoch, +): string { + const segment = (value: string): string => + `${value.length}:${value}`; + const path = reference.path.map((component) => + segment(`${component.quoted ? "q" : "u"}${component.value}`) + ).join(""); + const searchPaths = request.searchPaths.map((searchPath) => + segment(searchPath.map((component) => + segment(`${component.quoted ? "q" : "u"}${component.value}`) + ).join("")) + ).join(""); + return [ + segment(request.scope), + segment(request.dialectId), + segment(String(epoch.generation)), + segment(epoch.token), + segment(reference.relationEntityId ?? ""), + segment(path), + segment(searchPaths), + ].join(""); +} + +function cacheGet( + state: CoordinatorState, + key: string, +): ReadyRelation | null { + const value = state.cache.get(key); + if (!value) return null; + state.cache.delete(key); + state.cache.set(key, value); + return value; +} + +function cacheSet( + state: CoordinatorState, + key: string, + value: ReadyRelation, +): void { + state.cache.delete(key); + state.cache.set(key, value); + while (state.cache.size > state.maxCacheEntries) { + const oldest = state.cache.keys().next().value; + if (typeof oldest !== "string") break; + state.cache.delete(oldest); + } +} + +function withRequestKey( + relation: ReadyRelation, + requestKey: string, +): ReadyRelation { + return relation.requestKey === requestKey + ? relation + : Object.freeze({ ...relation, requestKey }); +} + +function settle( + consumer: ConsumerState, + outcome: SqlColumnCatalogBatchOutcome, +): void { + if (consumer.settled) return; + consumer.settled = true; + if (consumer.owner.active === consumer) { + consumer.owner.active = null; + } + const resolve = consumer.resolve; + consumer.resolve = null; + resolve?.(outcome); +} + +function cancelConsumer( + consumer: ConsumerState, + outcome: SqlColumnCatalogBatchOutcome, +): void { + if (consumer.settled) return; + consumer.controller.abort(); + settle(consumer, outcome); +} + +function makeSettledTicket( + outcome: SqlColumnCatalogBatchOutcome, +): SqlColumnCatalogBatchTicket { + return Object.freeze({ + cancel: (): void => {}, + result: Promise.resolve(outcome), + }); +} + +function combineRelations( + request: SqlColumnCatalogBatchRequest, + cached: ReadonlyMap, + loaded: readonly SqlColumnCatalogRelationResult[], +): readonly SqlColumnCatalogRelationResult[] | null { + const byKey = new Map(); + for (const relation of loaded) byKey.set(relation.requestKey, relation); + const output: SqlColumnCatalogRelationResult[] = []; + for (const reference of request.relations) { + const cachedRelation = cached.get(reference.requestKey); + const relation = cachedRelation + ? withRequestKey(cachedRelation, reference.requestKey) + : byKey.get(reference.requestKey); + if (!relation) return null; + output.push(relation); + } + return Object.freeze(output); +} + +function startProviderWork( + state: CoordinatorState, + owner: OwnerState, + fullRequest: SqlColumnCatalogBatchRequest, + missingRequest: SqlColumnCatalogBatchRequest, + cached: ReadonlyMap, + consumer: ConsumerState, +): void { + let providerResult: unknown; + try { + providerResult = state.context.loadColumns( + missingRequest, + consumer.controller.signal, + ); + } catch { + settle(consumer, unavailable("provider-failed")); + return; + } + Promise.resolve(providerResult).then( + (value) => { + if ( + consumer.settled || + consumer.controller.signal.aborted || + state.disposed || + owner.disposed || + owner.owner !== state + ) { + return; + } + const decoded = decodeSqlColumnCatalogBatchResponse( + state.capturedProvider, + missingRequest, + value, + ); + if (decoded.status === "malformed") { + settle(consumer, unavailable("malformed-response")); + return; + } + owner.observedEpoch = decoded.value.epoch; + for (const relation of decoded.value.relations) { + if ( + relation.status !== "ready" || + relation.coverage !== "complete" + ) continue; + const reference = missingRequest.relations.find( + (candidate) => candidate.requestKey === relation.requestKey, + ); + if (reference) { + cacheSet( + state, + cacheKey(missingRequest, reference, decoded.value.epoch), + relation, + ); + } + } + const relations = combineRelations( + fullRequest, + cached, + decoded.value.relations, + ); + settle( + consumer, + relations + ? Object.freeze({ + providerId: state.context.id, + epoch: decoded.value.epoch, + relations, + scope: owner.scope, + status: "usable", + }) + : unavailable("malformed-response"), + ); + }, + () => { + if (!consumer.settled) { + settle(consumer, unavailable("provider-failed")); + } + }, + ); +} + +function requestColumns( + owner: OwnerState, + input: unknown, +): SqlColumnCatalogBatchTicket { + const state = owner.owner; + if (!state || state.disposed || owner.disposed) { + return makeSettledTicket(unavailable("disposed")); + } + const expectedEpoch = dataProperty(input, "expectedEpoch"); + const relations = dataProperty(input, "relations"); + const searchPaths = dataProperty(input, "searchPaths"); + if (!expectedEpoch || !relations || !searchPaths) { + return makeSettledTicket(unavailable("invalid-request")); + } + const created = createSqlColumnCatalogBatchRequest({ + dialectId: owner.dialectId, + expectedEpoch: expectedEpoch.value === null + ? owner.observedEpoch + : expectedEpoch.value, + relations: relations.value, + scope: owner.scope, + searchPaths: searchPaths.value, + }); + if (created.status === "malformed") { + return makeSettledTicket(unavailable("invalid-request")); + } + const request = created.value; + if (owner.active) cancelConsumer(owner.active, SUPERSEDED); + + const cached = new Map(); + const missing: SqlColumnCatalogRelationReference[] = []; + for (const reference of request.relations) { + const key = request.expectedEpoch === null + ? null + : cacheKey(request, reference, request.expectedEpoch); + const cachedRelation = key === null ? null : cacheGet(state, key); + if (cachedRelation) { + cached.set(reference.requestKey, cachedRelation); + } else { + missing.push(reference); + } + } + if (missing.length === 0) { + const combined = combineRelations(request, cached, []); + if (request.expectedEpoch === null) { + return makeSettledTicket(unavailable("invalid-request")); + } + return makeSettledTicket( + Object.freeze({ + epoch: request.expectedEpoch, + providerId: state.context.id, + relations: combined ?? Object.freeze([]), + scope: owner.scope, + status: "usable", + }), + ); + } + const missingCreated = createSqlColumnCatalogBatchRequest({ + dialectId: request.dialectId, + expectedEpoch: request.expectedEpoch, + relations: missing, + scope: request.scope, + searchPaths: request.searchPaths, + }); + if (missingCreated.status === "malformed") { + return makeSettledTicket(unavailable("invalid-request")); + } + let resolveResult: ( + value: SqlColumnCatalogBatchOutcome, + ) => void = (): void => {}; + const result = new Promise((resolve) => { + resolveResult = resolve; + }); + const consumer: ConsumerState = { + controller: new AbortController(), + owner, + resolve: resolveResult, + settled: false, + }; + owner.active = consumer; + const ticket = Object.freeze({ + cancel: (): void => cancelConsumer(consumer, CANCELLED), + result, + }); + startProviderWork( + state, + owner, + request, + missingCreated.value, + cached, + consumer, + ); + return ticket; +} + +function disposeOwner(owner: OwnerState): void { + if (owner.disposed) return; + owner.disposed = true; + if (owner.active) { + cancelConsumer(owner.active, unavailable("disposed")); + } + owner.owner?.owners.delete(owner); + owner.owner = null; +} + +function prepareOwner( + state: CoordinatorState, + input: unknown, +): SqlColumnCatalogBatchOwnerResult { + if (state.disposed) { + return Object.freeze({ reason: "disposed", status: "unavailable" }); + } + const scopeProperty = dataProperty(input, "scope"); + const dialectProperty = dataProperty(input, "dialectId"); + const scope = boundedString( + scopeProperty?.value, + MAX_COLUMN_SCOPE_LENGTH, + ); + const dialectId = boundedString( + dialectProperty?.value, + MAX_COLUMN_DIALECT_ID_LENGTH, + ); + if (scope === null || dialectId === null) { + return Object.freeze({ + reason: "invalid-options", + status: "unavailable", + }); + } + const owner: OwnerState = { + active: null, + dialectId, + disposed: false, + owner: state, + observedEpoch: null, + scope, + }; + state.owners.add(owner); + return Object.freeze({ + owner: Object.freeze({ + dispose: (): void => disposeOwner(owner), + request: (input_: SqlColumnCatalogBatchInput) => + requestColumns(owner, input_), + }), + status: "prepared", + }); +} + +function disposeCoordinator(state: CoordinatorState): void { + if (state.disposed) return; + state.disposed = true; + state.cache.clear(); + for (const owner of state.owners) disposeOwner(owner); +} + +export function createSqlColumnCatalogBatchCoordinator( + input: unknown, +): SqlColumnCatalogBatchCoordinatorResult { + const providerProperty = dataProperty(input, "provider"); + if (!providerProperty) { + return Object.freeze({ + reason: "invalid-provider", + status: "unavailable", + }); + } + const captured = captureSqlColumnCatalogProvider( + providerProperty.value, + ); + if (captured.status === "malformed") { + return Object.freeze({ + reason: "invalid-provider", + status: "unavailable", + }); + } + const context = resolveSqlColumnCatalogProvider(captured.value); + if (!context) { + return Object.freeze({ + reason: "invalid-provider", + status: "unavailable", + }); + } + const maximumProperty = dataProperty(input, "maxCacheEntries"); + const maximum = maximumProperty?.value ?? DEFAULT_COLUMN_CACHE_ENTRIES; + if ( + typeof maximum !== "number" || + !Number.isSafeInteger(maximum) || + maximum < 1 || + maximum > MAX_COLUMN_CACHE_ENTRIES + ) { + return Object.freeze({ + reason: "invalid-options", + status: "unavailable", + }); + } + const state: CoordinatorState = { + cache: new Map(), + capturedProvider: captured.value, + context, + disposed: false, + maxCacheEntries: maximum, + owners: new Set(), + }; + return Object.freeze({ + coordinator: Object.freeze({ + dispose: (): void => disposeCoordinator(state), + prepareOwner: (options: SqlColumnCatalogOwnerOptions) => + prepareOwner(state, options), + providerId: context.id, + }), + status: "created", + }); +} diff --git a/src/vnext/column-catalog-boundary.ts b/src/vnext/column-catalog-boundary.ts new file mode 100644 index 0000000..179c00a --- /dev/null +++ b/src/vnext/column-catalog-boundary.ts @@ -0,0 +1,712 @@ +import type { SqlCatalogEpoch } from "./relation-completion-types.js"; +import type { + SqlColumnCatalogBatchRequest, + SqlColumnCatalogBatchResponse, + SqlColumnCatalogColumn, + SqlColumnCatalogRelationReference, + SqlColumnCatalogRelationResult, + SqlColumnCatalogResolvedColumn, +} from "./column-catalog-types.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, +} from "./types.js"; + +export const MAX_COLUMN_PROVIDER_ID_LENGTH = 256; +export const MAX_COLUMN_SCOPE_LENGTH = 512; +export const MAX_COLUMN_DIALECT_ID_LENGTH = 128; +export const MAX_COLUMN_EPOCH_TOKEN_LENGTH = 256; +export const MAX_COLUMN_ENTITY_ID_LENGTH = 256; +export const MAX_COLUMN_NAME_LENGTH = 256; +export const MAX_COLUMN_INSERT_TEXT_LENGTH = 1_024; +export const MAX_COLUMN_TYPE_LENGTH = 512; +export const MAX_COLUMN_DETAIL_LENGTH = 1_024; +export const MAX_COLUMN_BATCH_RELATIONS = 64; +export const MAX_COLUMN_RELATION_PATH_COMPONENTS = 32; +export const MAX_COLUMN_SEARCH_PATHS = 32; +export const MAX_COLUMN_SEARCH_PATH_COMPONENTS = 8; +export const MAX_COLUMNS_PER_RELATION = 256; +export const MAX_COLUMNS_PER_BATCH = 4_096; + +const capturedColumnProviders = new WeakMap< + object, + { + readonly id: string; + readonly loadColumns: Function; + } +>(); +const capturedColumnProviderBrand: unique symbol = Symbol( + "CapturedSqlColumnCatalogProvider", +); + +export interface CapturedSqlColumnCatalogProvider { + readonly [capturedColumnProviderBrand]: + "CapturedSqlColumnCatalogProvider"; +} + +export interface SqlCapturedColumnCatalogProviderContext { + readonly id: string; + readonly loadColumns: ( + this: void, + request: SqlColumnCatalogBatchRequest, + signal: AbortSignal, + ) => unknown; +} + +export type SqlColumnBoundaryFailureReason = + | "duplicate-column-entity-id" + | "duplicate-request-key" + | "invalid-shape" + | "resource-limit" + | "unexpected-relation"; + +export type SqlColumnBoundaryResult = + | { + readonly status: "accepted"; + readonly value: Value; + } + | { + readonly reason: SqlColumnBoundaryFailureReason; + readonly status: "malformed"; + }; + +interface DataRecord { + readonly fields: ReadonlyMap; +} + +const FAILURE_CODES = new Set([ + "authentication", + "authorization", + "invalid-configuration", + "rate-limited", + "unavailable", + "unknown", +]); +const RETRY_POLICIES = new Set([ + "after-invalidation", + "never", + "next-request", +]); + +type FailureCode = Extract< + SqlColumnCatalogRelationResult, + { readonly status: "failed" } +>["code"]; +type RetryPolicy = Extract< + SqlColumnCatalogRelationResult, + { readonly status: "failed" } +>["retry"]; + +function isFailureCode(value: unknown): value is FailureCode { + return typeof value === "string" && FAILURE_CODES.has(value); +} + +function isRetryPolicy(value: unknown): value is RetryPolicy { + return typeof value === "string" && RETRY_POLICIES.has(value); +} + +function accepted( + value: Value, +): SqlColumnBoundaryResult { + return Object.freeze({ status: "accepted", value }); +} + +function malformed( + reason: SqlColumnBoundaryFailureReason, +): SqlColumnBoundaryResult { + return Object.freeze({ reason, status: "malformed" }); +} + +function readRecord( + value: unknown, + allowedKeys: ReadonlySet, +): DataRecord | null { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + return null; + } + let keys: readonly PropertyKey[]; + try { + keys = Reflect.ownKeys(value); + } catch { + return null; + } + const fields = new Map(); + for (const key of keys) { + if (typeof key !== "string" || !allowedKeys.has(key)) return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, key); + } catch { + return null; + } + if (!descriptor || !("value" in descriptor)) return null; + fields.set(key, descriptor.value); + } + return { fields }; +} + +function required(record: DataRecord, key: string): unknown { + return record.fields.has(key) ? record.fields.get(key) : undefined; +} + +function boundedString(value: unknown, maximum: number): string | null { + return typeof value === "string" && + value.length > 0 && + value.length <= maximum + ? value + : null; +} + +function optionalBoundedString( + value: unknown, + maximum: number, +): string | undefined | null { + return value === undefined + ? undefined + : boundedString(value, maximum); +} + +function decodeEpoch(value: unknown): SqlCatalogEpoch | null { + const record = readRecord( + value, + new Set(["generation", "token"]), + ); + if (!record) return null; + const generation = required(record, "generation"); + const token = boundedString( + required(record, "token"), + MAX_COLUMN_EPOCH_TOKEN_LENGTH, + ); + if ( + typeof generation !== "number" || + !Number.isSafeInteger(generation) || + generation < 0 || + token === null + ) { + return null; + } + return Object.freeze({ generation, token }); +} + +function decodeExpectedEpoch( + value: unknown, +): SqlCatalogEpoch | null | undefined { + return value === null ? null : decodeEpoch(value) ?? undefined; +} + +function readArrayElement( + values: readonly unknown[], + index: number, +): { readonly found: true; readonly value: unknown } | null { + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(values, index); + } catch { + return null; + } + return descriptor && "value" in descriptor + ? { found: true, value: descriptor.value } + : null; +} + +function readArrayLength(value: unknown, maximum: number): number | null { + if (!Array.isArray(value)) return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, "length"); + } catch { + return null; + } + if ( + !descriptor || + !("value" in descriptor) || + typeof descriptor.value !== "number" || + !Number.isSafeInteger(descriptor.value) || + descriptor.value < 0 || + descriptor.value > maximum + ) { + return null; + } + return descriptor.value; +} + +function decodeIdentifierPath( + value: unknown, + maximumComponents: number, +): SqlIdentifierPath | null { + const length = readArrayLength(value, maximumComponents); + if (length === null || length === 0 || !Array.isArray(value)) return null; + const output: SqlIdentifierComponent[] = []; + for (let index = 0; index < length; index += 1) { + const item = readArrayElement(value, index); + if (!item) return null; + const record = readRecord( + item.value, + new Set(["quoted", "value"]), + ); + if (!record) return null; + const identifier = boundedString( + required(record, "value"), + MAX_COLUMN_NAME_LENGTH, + ); + const quoted = required(record, "quoted"); + if (identifier === null || typeof quoted !== "boolean") return null; + output.push(Object.freeze({ quoted, value: identifier })); + } + return Object.freeze(output); +} + +function decodeSearchPaths(value: unknown): readonly SqlIdentifierPath[] | null { + const length = readArrayLength(value, MAX_COLUMN_SEARCH_PATHS); + if (length === null || !Array.isArray(value)) return null; + const paths: SqlIdentifierPath[] = []; + for (let index = 0; index < length; index += 1) { + const item = readArrayElement(value, index); + if (!item) return null; + const path = decodeIdentifierPath( + item.value, + MAX_COLUMN_SEARCH_PATH_COMPONENTS, + ); + if (!path) return null; + paths.push(path); + } + return Object.freeze(paths); +} + +function decodeRelationReference( + value: unknown, +): SqlColumnCatalogRelationReference | null { + const record = readRecord( + value, + new Set(["path", "relationEntityId", "requestKey"]), + ); + if (!record) return null; + const requestKey = boundedString( + required(record, "requestKey"), + MAX_COLUMN_ENTITY_ID_LENGTH, + ); + const path = decodeIdentifierPath( + required(record, "path"), + MAX_COLUMN_RELATION_PATH_COMPONENTS, + ); + const relationEntityId = optionalBoundedString( + required(record, "relationEntityId"), + MAX_COLUMN_ENTITY_ID_LENGTH, + ); + if (requestKey === null || !path || relationEntityId === null) return null; + return Object.freeze({ + path, + requestKey, + ...(relationEntityId === undefined ? {} : { relationEntityId }), + }); +} + +function sameEpoch(left: SqlCatalogEpoch, right: SqlCatalogEpoch): boolean { + return left.generation === right.generation && left.token === right.token; +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +export function captureSqlColumnCatalogProvider( + provider: unknown, +): SqlColumnBoundaryResult { + const record = readRecord(provider, new Set(["id", "loadColumns"])); + if (!record) return malformed("invalid-shape"); + const id = boundedString( + required(record, "id"), + MAX_COLUMN_PROVIDER_ID_LENGTH, + ); + const loadColumns = required(record, "loadColumns"); + if (id === null || typeof loadColumns !== "function") { + return malformed("invalid-shape"); + } + const capturedValue: CapturedSqlColumnCatalogProvider = { + [capturedColumnProviderBrand]: "CapturedSqlColumnCatalogProvider", + }; + const captured = Object.freeze(capturedValue); + capturedColumnProviders.set(captured, { id, loadColumns }); + return accepted(captured); +} + +export function resolveSqlColumnCatalogProvider( + provider: unknown, +): SqlCapturedColumnCatalogProviderContext | null { + if (provider === null || typeof provider !== "object") return null; + const captured = capturedColumnProviders.get(provider); + if (!captured) return null; + return Object.freeze({ + id: captured.id, + loadColumns: ( + request: SqlColumnCatalogBatchRequest, + signal: AbortSignal, + ): unknown => + Reflect.apply( + captured.loadColumns, + undefined, + [request, signal], + ), + }); +} + +export function createSqlColumnCatalogBatchRequest( + input: unknown, +): SqlColumnBoundaryResult { + const record = readRecord( + input, + new Set([ + "dialectId", + "expectedEpoch", + "relations", + "scope", + "searchPaths", + ]), + ); + if (!record) return malformed("invalid-shape"); + const scope = boundedString( + required(record, "scope"), + MAX_COLUMN_SCOPE_LENGTH, + ); + const dialectId = boundedString( + required(record, "dialectId"), + MAX_COLUMN_DIALECT_ID_LENGTH, + ); + const expectedEpoch = decodeExpectedEpoch( + required(record, "expectedEpoch"), + ); + const relationsValue = required(record, "relations"); + const searchPaths = decodeSearchPaths(required(record, "searchPaths")); + const relationCount = readArrayLength( + relationsValue, + MAX_COLUMN_BATCH_RELATIONS, + ); + if ( + scope === null || + dialectId === null || + expectedEpoch === undefined || + relationCount === null || + !Array.isArray(relationsValue) || + searchPaths === null + ) { + return malformed("invalid-shape"); + } + const keys = new Set(); + const relations: SqlColumnCatalogRelationReference[] = []; + for (let index = 0; index < relationCount; index += 1) { + const item = readArrayElement(relationsValue, index); + if (!item) return malformed("invalid-shape"); + const reference = decodeRelationReference(item.value); + if (!reference) return malformed("invalid-shape"); + if (keys.has(reference.requestKey)) { + return malformed("duplicate-request-key"); + } + keys.add(reference.requestKey); + relations.push(reference); + } + if (relations.length === 0) return malformed("invalid-shape"); + relations.sort((left, right) => + compareText(left.requestKey, right.requestKey) + ); + return accepted(Object.freeze({ + dialectId, + expectedEpoch, + relations: Object.freeze(relations), + searchPaths, + scope, + })); +} + +function decodeColumn( + value: unknown, + providerId: string, + request: SqlColumnCatalogBatchRequest, + epoch: SqlCatalogEpoch, + relationEntityId: string, +): SqlColumnCatalogResolvedColumn | null { + const record = readRecord( + value, + new Set([ + "columnEntityId", + "dataType", + "detail", + "identifier", + "insertText", + "ordinal", + ]), + ); + if (!record) return null; + const columnEntityId = boundedString( + required(record, "columnEntityId"), + MAX_COLUMN_ENTITY_ID_LENGTH, + ); + const identifierRecord = readRecord( + required(record, "identifier"), + new Set(["quoted", "value"]), + ); + const identifierValue = identifierRecord + ? boundedString( + required(identifierRecord, "value"), + MAX_COLUMN_NAME_LENGTH, + ) + : null; + const quoted = identifierRecord + ? required(identifierRecord, "quoted") + : null; + const insertText = boundedString( + required(record, "insertText"), + MAX_COLUMN_INSERT_TEXT_LENGTH, + ); + const ordinal = required(record, "ordinal"); + const dataType = optionalBoundedString( + required(record, "dataType"), + MAX_COLUMN_TYPE_LENGTH, + ); + const detail = optionalBoundedString( + required(record, "detail"), + MAX_COLUMN_DETAIL_LENGTH, + ); + if ( + columnEntityId === null || + identifierValue === null || + typeof quoted !== "boolean" || + insertText === null || + dataType === null || + detail === null || + typeof ordinal !== "number" || + !Number.isSafeInteger(ordinal) || + ordinal < 0 + ) { + return null; + } + const column: SqlColumnCatalogColumn = { + columnEntityId, + identifier: Object.freeze({ + quoted, + value: identifierValue, + }), + insertText, + ordinal, + ...(dataType === undefined ? {} : { dataType }), + ...(detail === undefined ? {} : { detail }), + }; + return Object.freeze({ + ...column, + provenance: Object.freeze({ + columnEntityId, + epoch, + providerId, + relationEntityId, + scope: request.scope, + }), + }); +} + +function columnKey(column: SqlColumnCatalogResolvedColumn): string { + return [ + column.columnEntityId, + column.ordinal, + column.identifier.quoted ? "q" : "u", + column.identifier.value, + column.insertText, + column.dataType ?? "", + column.detail ?? "", + ].join("\u0000"); +} + +function decodeReadyRelation( + record: DataRecord, + providerId: string, + request: SqlColumnCatalogBatchRequest, + epoch: SqlCatalogEpoch, + requestKey: string, + relationEntityId: string, +): SqlColumnBoundaryResult { + const coverage = required(record, "coverage"); + const columnsValue = required(record, "columns"); + const columnCount = readArrayLength( + columnsValue, + MAX_COLUMNS_PER_RELATION, + ); + if ( + (coverage !== "complete" && coverage !== "partial") || + columnCount === null || + !Array.isArray(columnsValue) + ) { + return malformed("invalid-shape"); + } + const byId = new Map(); + for (let index = 0; index < columnCount; index += 1) { + const item = readArrayElement(columnsValue, index); + if (!item) return malformed("invalid-shape"); + const column = decodeColumn( + item.value, + providerId, + request, + epoch, + relationEntityId, + ); + if (!column) return malformed("invalid-shape"); + const previous = byId.get(column.columnEntityId); + if (previous && columnKey(previous) !== columnKey(column)) { + return malformed("duplicate-column-entity-id"); + } + byId.set(column.columnEntityId, column); + } + const columns = [...byId.values()].sort((left, right) => + left.ordinal - right.ordinal || + compareText(left.identifier.value, right.identifier.value) || + compareText(left.columnEntityId, right.columnEntityId) + ); + return accepted(Object.freeze({ + columns: Object.freeze(columns), + coverage, + relationEntityId, + requestKey, + status: "ready", + })); +} + +function decodeRelation( + value: unknown, + providerId: string, + request: SqlColumnCatalogBatchRequest, + epoch: SqlCatalogEpoch, +): SqlColumnBoundaryResult { + const record = readRecord( + value, + new Set([ + "code", + "columns", + "coverage", + "relationEntityId", + "requestKey", + "retry", + "status", + ]), + ); + if (!record) return malformed("invalid-shape"); + const requestKey = boundedString( + required(record, "requestKey"), + MAX_COLUMN_ENTITY_ID_LENGTH, + ); + const status = required(record, "status"); + if (requestKey === null) return malformed("invalid-shape"); + if (status === "ready") { + const relationEntityId = boundedString( + required(record, "relationEntityId"), + MAX_COLUMN_ENTITY_ID_LENGTH, + ); + if (relationEntityId === null) return malformed("invalid-shape"); + return decodeReadyRelation( + record, + providerId, + request, + epoch, + requestKey, + relationEntityId, + ); + } + if (status === "loading") { + return accepted(Object.freeze({ + requestKey, + status, + })); + } + const code = required(record, "code"); + const retry = required(record, "retry"); + if ( + status !== "failed" || + !isFailureCode(code) || + !isRetryPolicy(retry) + ) { + return malformed("invalid-shape"); + } + return accepted(Object.freeze({ + code, + requestKey, + retry, + status, + })); +} + +export function decodeSqlColumnCatalogBatchResponse( + provider: CapturedSqlColumnCatalogProvider, + request: SqlColumnCatalogBatchRequest, + value: unknown, +): SqlColumnBoundaryResult { + const context = resolveSqlColumnCatalogProvider(provider); + const record = readRecord(value, new Set(["epoch", "relations"])); + if (!context || !record) return malformed("invalid-shape"); + const epoch = decodeEpoch(required(record, "epoch")); + const relationsValue = required(record, "relations"); + const relationCount = readArrayLength( + relationsValue, + MAX_COLUMN_BATCH_RELATIONS, + ); + if ( + epoch === null || + (request.expectedEpoch !== null && + !sameEpoch(epoch, request.expectedEpoch)) || + relationCount === null || + !Array.isArray(relationsValue) + ) { + return malformed("invalid-shape"); + } + if (relationCount > request.relations.length) { + return malformed("resource-limit"); + } + const requested = new Set( + request.relations.map((relation) => relation.requestKey), + ); + const references = new Map( + request.relations.map((relation) => [ + relation.requestKey, + relation, + ]), + ); + const byId = new Map(); + let columnCount = 0; + for (let index = 0; index < relationCount; index += 1) { + const item = readArrayElement(relationsValue, index); + if (!item) return malformed("invalid-shape"); + const decoded = decodeRelation( + item.value, + context.id, + request, + epoch, + ); + if (decoded.status === "malformed") return decoded; + const relation = decoded.value; + if (!requested.has(relation.requestKey)) { + return malformed("unexpected-relation"); + } + const reference = references.get(relation.requestKey); + if ( + relation.status === "ready" && + reference?.relationEntityId !== undefined && + relation.relationEntityId !== reference.relationEntityId + ) { + return malformed("invalid-shape"); + } + if (byId.has(relation.requestKey)) { + return malformed("duplicate-request-key"); + } + if (relation.status === "ready") { + columnCount += relation.columns.length; + if (columnCount > MAX_COLUMNS_PER_BATCH) { + return malformed("resource-limit"); + } + } + byId.set(relation.requestKey, relation); + } + if (byId.size !== requested.size) return malformed("invalid-shape"); + return accepted(Object.freeze({ + epoch, + relations: Object.freeze( + [...byId.values()].sort((left, right) => + compareText(left.requestKey, right.requestKey) + ), + ), + })); +} diff --git a/src/vnext/column-catalog-types.ts b/src/vnext/column-catalog-types.ts new file mode 100644 index 0000000..8cf9e8c --- /dev/null +++ b/src/vnext/column-catalog-types.ts @@ -0,0 +1,82 @@ +import type { SqlCatalogEpoch } from "./relation-completion-types.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, +} from "./types.js"; + +export type SqlColumnCatalogCoverage = "complete" | "partial"; + +export interface SqlColumnCatalogRelationReference { + readonly path: SqlIdentifierPath; + readonly relationEntityId?: string; + readonly requestKey: string; +} + +export interface SqlColumnCatalogBatchRequest { + readonly dialectId: string; + readonly expectedEpoch: SqlCatalogEpoch | null; + readonly relations: readonly SqlColumnCatalogRelationReference[]; + readonly searchPaths: readonly SqlIdentifierPath[]; + readonly scope: string; +} + +export interface SqlColumnCatalogColumn { + readonly columnEntityId: string; + readonly dataType?: string; + readonly detail?: string; + readonly identifier: SqlIdentifierComponent; + readonly insertText: string; + readonly ordinal: number; +} + +export interface SqlColumnCatalogProvenance { + readonly columnEntityId: string; + readonly epoch: SqlCatalogEpoch; + readonly providerId: string; + readonly relationEntityId: string; + readonly scope: string; +} + +export interface SqlColumnCatalogResolvedColumn + extends SqlColumnCatalogColumn { + readonly provenance: SqlColumnCatalogProvenance; +} + +export type SqlColumnCatalogRelationResult = + | { + readonly columns: readonly SqlColumnCatalogResolvedColumn[]; + readonly coverage: SqlColumnCatalogCoverage; + readonly relationEntityId: string; + readonly requestKey: string; + readonly status: "ready"; + } + | { + readonly requestKey: string; + readonly status: "loading"; + } + | { + readonly code: + | "authentication" + | "authorization" + | "invalid-configuration" + | "rate-limited" + | "unavailable" + | "unknown"; + readonly requestKey: string; + readonly retry: "after-invalidation" | "never" | "next-request"; + readonly status: "failed"; + }; + +export interface SqlColumnCatalogBatchResponse { + readonly epoch: SqlCatalogEpoch; + readonly relations: readonly SqlColumnCatalogRelationResult[]; +} + +export interface SqlColumnCatalogProvider { + readonly id: string; + readonly loadColumns: ( + this: void, + request: SqlColumnCatalogBatchRequest, + signal: AbortSignal, + ) => Promise; +} From 8f9f39964d281669c809356d45e0dcf991d903fb Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:47:21 +0800 Subject: [PATCH 06/25] docs(vnext): model marimo column migration --- docs/vnext/marimo-sql-migration.md | 85 ++++++--- .../marimo-sql-migration.test-d.ts | 164 +++++++++++++++++- 2 files changed, 222 insertions(+), 27 deletions(-) diff --git a/docs/vnext/marimo-sql-migration.md b/docs/vnext/marimo-sql-migration.md index 09afc39..fc845d9 100644 --- a/docs/vnext/marimo-sql-migration.md +++ b/docs/vnext/marimo-sql-migration.md @@ -1,6 +1,6 @@ # Marimo SQL Completion Migration -Status: implementation fixture; relation completion only +Status: implementation fixture; relation and column completion The compile-only fixture [`marimo-sql-migration.test-d.ts`](../../test/vnext-types/marimo-sql-migration.test-d.ts) @@ -14,8 +14,9 @@ Every `sqlEditor` support/view opens its own session against that service. Destroying a view never disposes the service; the application disposes the service after its views have been retired. -One provider projects `dataConnectionsMapAtom` and `datasetTablesAtom` into -canonical relation records. It must preserve today's collision rule: +One relation provider projects `dataConnectionsMapAtom` and +`datasetTablesAtom` into canonical relation records. It must preserve today's +collision rule: connection relations win over same-named local relations. Provider records contain plain immutable data and stable entity IDs, never Jotai atoms, React roots, or backend objects. @@ -32,6 +33,27 @@ epoch and notify the one scoped subscription. The context `searchPath` values reproduce `default_database`, `default_schema`, nested-schema, and schemaless behavior. +## Batched DataTable columns + +The same shared service configures one `SqlColumnCatalogProvider`. Each +`loadColumns` call forwards the complete relation request array to one +DataTable metadata batch operation. It does not fetch once per relation. +Relation IDs are the stable IDs emitted by the relation provider. Column IDs +are stable within that relation and connection incarnation. + +Each column supplies a canonical identifier and provider-rendered +`insertText`; these are intentionally separate so quoted or dialect-sensitive +names insert correctly. The provider also supplies ordinal, data type, and +detail when known. A cold request carries `expectedEpoch: null`. The returned +epoch becomes the authority for later work, while connection replacement gets +a new scope. + +Every requested relation independently reports complete or partial ready +columns, loading, or normalized failure. Partial, loading, and failed results +remain retryable according to the library contract and are not disguised as +complete empty schemas. The public boundary validates the provider's unknown +payload before it reaches completion composition. + ## One atomic editor input Marimo should have one transaction builder for SQL interpretation changes. A @@ -72,8 +94,9 @@ The gate receives only the immutable CodeMirror `EditorState` and numeric position. It must remain synchronous and side-effect free. Do not encode a range beyond the document or an empty embedded region. -The rich-info resolver uses catalog provenance IDs to look up current marimo -metadata, mounts React into a new element, and returns +The rich-info resolver uses relation `catalog` provenance or +`column-catalog` scope, relation ID, and column ID to look up current marimo +metadata. It mounts React into a new element and returns `{ dom, destroy: root.unmount }`. The adapter owns cancellation and disposal. ## Dialect cutover @@ -98,42 +121,58 @@ explicit, tested generic-dialect policy. Marimo's current `tablesCompletionSource()` is broader than its name. The CodeMirror SQL schema source provides relations, namespace navigation, and -columns. vNext currently provides relation items only. +columns. vNext now provides the relation and lazy batched column portions, +including qualified and unqualified query-site completion, ambiguity handling, +stable provenance, cancellation, and bounded provider work. -No-regression replacement therefore still requires: +The remaining feature gaps are: -- lazy, bounded, cancellation-aware column lookup keyed by stable relation - entity IDs; -- qualified and unqualified column completion with aliases, joins, - ambiguity, quoting, and exact edits; -- column provenance for the disposable info resolver; -- namespace/container parity for database, schema, project, and dataset +- namespace/container completion for database, schema, project, and dataset navigation; and - the dialect coverage described above. -Column lookup should be batched for all visible relations at a completion site. -It must not attach every column to every relation-search response or issue one -provider request per relation. +The fixture defines marimo's immutable namespace projection—stable entity ID, +scope, canonical identifier path, and namespace kind—but deliberately does +not invent a provider import. Once a public namespace provider lands, that +projection should feed one scoped provider on the shared service. ## Migration sequence 1. Capture golden legacy results for labels, kinds, edit ranges, qualification, and details across representative connection shapes. -2. Land the shared provider, incarnation scope, atomic transaction builder, - region scanner, insertion-point completion gate, and completion router - behind a feature flag. +2. Land the shared relation and batched column providers, incarnation scope, + atomic transaction builder, region scanner, insertion-point completion + gate, and completion router behind a feature flag. 3. Run vNext relation completion in shadow mode while the legacy source remains visible. -4. Add column and namespace completion to the library and compare the golden - corpus. -5. Cut over the four supported dialects as one source replacement, preserving +4. Compare relation and column results with the golden corpus, including + quoted insert text, aliases, ambiguity, partial/loading/failure states, and + cold epoch behavior. +5. Add the public namespace provider and connect the prepared marimo namespace + projection. +6. Cut over the four supported dialects as one source replacement, preserving variable and keyword external sources. -6. Add dialect coverage, expand the router, and remove completion-only legacy +7. Add dialect coverage, expand the router, and remove completion-only legacy schema code. Keep legacy schema data while hover or diagnostics still use it. ## Acceptance tests +The compile-only marimo fixture proves: + +- one shared service configured with one relation provider and one batched + column provider; +- a two-relation cold column request with `expectedEpoch: null`; +- stable relation and column IDs, canonical identifiers, and distinct + provider-rendered insert text; +- partial, loading, and failure provider states; +- relation and column provenance lookup by current scope with a React + disposer; +- atomic editor interpretation updates, complete template regions, and the + unmatched-expression insertion-point gate; +- preservation of variable and keyword sources; and +- supported-dialect vNext routing with exclusive legacy fallback. + Library tests cover relation and column completion for `FROM`, `JOIN`, `alias.`, unqualified projections and predicates, `USING`, nested queries, CTEs, quoted identifiers, ambiguous columns, provider loading/invalidations, diff --git a/test/vnext-types/marimo-sql-migration.test-d.ts b/test/vnext-types/marimo-sql-migration.test-d.ts index b116f3a..740e012 100644 --- a/test/vnext-types/marimo-sql-migration.test-d.ts +++ b/test/vnext-types/marimo-sql-migration.test-d.ts @@ -20,9 +20,12 @@ import { type SqlCatalogRelation, type SqlCatalogSearchRequest, type SqlCatalogSearchResponse, + type SqlColumnCatalogBatchRequest, + type SqlColumnCatalogProvider, type SqlContextInput, type SqlDocumentContext, type SqlEmbeddedRegion, + type SqlIdentifierComponent, type SqlIdentifierPath, type SqlRelationCatalogProvider, } from "../../src/vnext/index.js"; @@ -64,15 +67,71 @@ interface MarimoTableMetadata { readonly entityId: string; } +interface MarimoColumnMetadata { + readonly columnEntityId: string; + readonly detail: string; + readonly relationEntityId: string; +} + interface MarimoCatalogSnapshot { readonly epoch: SqlCatalogEpoch; readonly relations: readonly SqlCatalogRelation[]; } +interface MarimoDataTableColumn { + readonly columnEntityId: string; + readonly dataType?: string; + readonly detail?: string; + readonly identifier: SqlIdentifierComponent; + readonly insertText: string; + readonly ordinal: number; +} + +type MarimoDataTableColumnResult = + | { + readonly columns: readonly MarimoDataTableColumn[]; + readonly coverage: "complete" | "partial"; + readonly relationEntityId: string; + readonly requestKey: string; + readonly status: "ready"; + } + | { + readonly requestKey: string; + readonly status: "loading"; + } + | { + readonly code: + | "authentication" + | "authorization" + | "invalid-configuration" + | "rate-limited" + | "unavailable" + | "unknown"; + readonly requestKey: string; + readonly retry: "after-invalidation" | "never" | "next-request"; + readonly status: "failed"; + }; + +interface MarimoDataTableColumnBatch { + readonly epoch: SqlCatalogEpoch; + readonly relations: readonly MarimoDataTableColumnResult[]; +} + +interface MarimoNamespaceProjection { + readonly entityId: string; + readonly kind: "catalog" | "dataset" | "project" | "schema"; + readonly path: SqlIdentifierPath; + readonly scope: string; +} + declare const catalogByScope: ReadonlyMap; declare const tableMetadataById: ReadonlyMap; +declare const columnMetadataByProvenance: + ReadonlyMap; +declare const namespaceProjectionByScope: + ReadonlyMap; declare const subscribeToCatalogScope: ( scope: string, listener: (epoch: SqlCatalogEpoch) => void, @@ -84,6 +143,10 @@ declare const searchMarimoCatalogIndex: ( readonly coverage: SqlCatalogReadyCoverage; readonly relations: readonly SqlCatalogRelation[]; }; +declare const loadMarimoDataTableColumnBatch: ( + request: SqlColumnCatalogBatchRequest, + signal: AbortSignal, +) => Promise; declare const variableCompletionSource: CompletionSource; declare const keywordCompletionSource: CompletionSource; declare const legacySchemaCompletionSource: CompletionSource; @@ -167,10 +230,44 @@ const marimoCatalogProvider: SqlRelationCatalogProvider = { }, }; +const marimoColumnProvider: SqlColumnCatalogProvider = { + id: "marimo-datatable-columns", + loadColumns: async (request, signal) => { + signal.throwIfAborted(); + const batch = await loadMarimoDataTableColumnBatch(request, signal); + signal.throwIfAborted(); + return { + epoch: batch.epoch, + relations: batch.relations.map((result) => { + if (result.status !== "ready") return result; + return { + columns: result.columns.map((column) => ({ + columnEntityId: column.columnEntityId, + identifier: column.identifier, + insertText: column.insertText, + ordinal: column.ordinal, + ...(column.dataType === undefined + ? {} + : { dataType: column.dataType }), + ...(column.detail === undefined + ? {} + : { detail: column.detail }), + })), + coverage: result.coverage, + relationEntityId: result.relationEntityId, + requestKey: result.requestKey, + status: result.status, + }; + }), + }; + }, +}; + // One caller-owned service is shared by every SQL editor support/view. const sharedSqlService = createSqlLanguageService({ catalog: marimoCatalogProvider, + columns: marimoColumnProvider, dialects: [ bigQueryDialect(), dremioDialect(), @@ -184,10 +281,13 @@ const infoResolver: SqlCompletionInfoResolver = ( { signal }, ) => { signal.throwIfAborted(); - if (item.provenance.kind !== "catalog") return null; - const metadata = tableMetadataById.get( - item.provenance.entityId, - ); + const metadata = item.provenance.kind === "catalog" + ? tableMetadataById.get(item.provenance.entityId) + : item.provenance.kind === "column-catalog" + ? columnMetadataByProvenance.get( + `${item.provenance.scope}\0${item.provenance.relationEntityId}\0${item.provenance.columnEntityId}`, + ) + : undefined; if (metadata === undefined) return null; const dom = document.createElement("div"); const root = createReactRoot(dom); @@ -387,9 +487,65 @@ const relationPath = [ { quoted: false, role: "relation", value: "users" }, ] satisfies SqlCanonicalRelationPath; +const coldColumnBatch = { + dialectId: "duckdb", + expectedEpoch: null, + relations: [ + { + path: [ + { quoted: false, value: "main" }, + { quoted: false, value: "users" }, + ], + relationEntityId: "connection:users", + requestKey: "binding:users", + }, + { + path: [ + { quoted: false, value: "main" }, + { quoted: false, value: "orders" }, + ], + relationEntityId: "connection:orders", + requestKey: "binding:orders", + }, + ], + searchPaths: [[{ quoted: false, value: "main" }]], + scope: "duckdb:42", +} satisfies SqlColumnCatalogBatchRequest; + +const columnProviderStateExamples = [ + { + columns: [{ + columnEntityId: "connection:users:customer-id", + dataType: "VARCHAR", + detail: "Customer identifier", + identifier: { quoted: true, value: "customer id" }, + insertText: '"customer id"', + ordinal: 0, + }], + coverage: "partial", + relationEntityId: "connection:users", + requestKey: "binding:users", + status: "ready", + }, + { + requestKey: "binding:orders", + status: "loading", + }, + { + code: "authorization", + requestKey: "binding:private-orders", + retry: "after-invalidation", + status: "failed", + }, +] satisfies readonly MarimoDataTableColumnResult[]; + void atomicSwitch; +void coldColumnBatch; +void columnProviderStateExamples; void firstSupport; void marimoCatalogProvider; +void marimoColumnProvider; +void namespaceProjectionByScope; void relationPath; void secondSupport; From 1fafcee4cf3a19ef03bb9af0e5aee4ee33aed883 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:50:20 +0800 Subject: [PATCH 07/25] feat(vnext): normalize parser query bindings --- docs/adr/0003-node-sql-parser-adapter.md | 12 +- docs/adr/0004-isolated-parser-execution.md | 20 +- docs/adr/0006-query-binding-model.md | 7 +- .../__tests__/node-sql-parser-adapter.test.ts | 45 +- .../node-sql-parser-browser-executor.test.ts | 24 +- ...sql-parser-browser-worker-endpoint.test.ts | 44 +- .../node-sql-parser-query-bindings.bench.ts | 31 + .../node-sql-parser-query-bindings.test.ts | 658 ++++++++++++ .../__tests__/node-sql-parser-wire.test.ts | 42 +- .../fixtures/node-sql-parser-crash-worker.js | 2 +- .../fixtures/node-sql-parser-silent-worker.js | 2 +- .../node-sql-parser-browser-executor.test.ts | 5 + src/vnext/node-sql-parser-adapter.ts | 38 +- src/vnext/node-sql-parser-browser-executor.ts | 3 + ...node-sql-parser-browser-worker-endpoint.ts | 24 + src/vnext/node-sql-parser-query-bindings.ts | 984 ++++++++++++++++++ src/vnext/node-sql-parser-wire.ts | 97 +- src/vnext/query-binding-model.ts | 6 +- 18 files changed, 1962 insertions(+), 82 deletions(-) create mode 100644 src/vnext/__tests__/node-sql-parser-query-bindings.bench.ts create mode 100644 src/vnext/__tests__/node-sql-parser-query-bindings.test.ts create mode 100644 src/vnext/node-sql-parser-query-bindings.ts diff --git a/docs/adr/0003-node-sql-parser-adapter.md b/docs/adr/0003-node-sql-parser-adapter.md index 3e60f1f..15f21c5 100644 --- a/docs/adr/0003-node-sql-parser-adapter.md +++ b/docs/adr/0003-node-sql-parser-adapter.md @@ -145,11 +145,11 @@ A direct object or one-element array is accepted. Zero or multiple roots are `failed/malformed-output`. The public normalized artifact exposes only its closed statement kind and full -statement-relative range. The backend AST is retained in a module-private -`WeakMap` keyed by the authenticated artifact. It is neither enumerable nor -returned through the syntax contract. Future relation extraction must decode -and validate backend nodes inside the adapter boundary rather than exposing -the raw AST to core features. +statement-relative range. The adapter consumes the backend AST immediately +through the bounded query-binding decoder and retains only an immutable, +backend-neutral binding model in a module-private `WeakMap` keyed by the +authenticated artifact. The raw AST is not retained, enumerable, or returned +through the syntax contract. ### Cancellation and execution placement @@ -201,7 +201,7 @@ This decision does not define: - Stable parser or provider APIs - Session cache keys or eviction - Document-level syntax diagnostics -- Semantic relation, scope, column, or type models +- Public semantic relation, scope, column, or type models - A worker protocol - Native DuckDB or remote validation providers - Dremio parsing diff --git a/docs/adr/0004-isolated-parser-execution.md b/docs/adr/0004-isolated-parser-execution.md index c5556cb..4312273 100644 --- a/docs/adr/0004-isolated-parser-execution.md +++ b/docs/adr/0004-isolated-parser-execution.md @@ -141,9 +141,10 @@ The initial request contains only: DuckDB uses the PostgreSQL grammar. Target-dialect policy stays in the main realm. -The initial response contains only one closed outcome: +Protocol v2 contains only one closed outcome: -- Parsed normalized statement kind +- Parsed normalized statement kind and a bounded query-binding DTO, or `null` + when the statement has no supported query-binding evidence - Syntax rejection - Bounded unsupported reason - Bounded failure code; retryability is derived from that code @@ -183,16 +184,17 @@ settles the active operation exactly once without exposing raw event data. Raw backend ASTs will not cross the worker boundary and the first protocol will not introduce remote AST handles or worker-local AST leases. -The first relation-completion slice is parser-independent under +The first relation-completion slice remains parser-independent under [ADR 0005](0005-parser-independent-relation-completion.md). Protocol v1 remains -syntax-only. Incomplete relation completion must not wait for worker startup, +historical and protocol v2 adds the internal query-binding DTO. Incomplete +relation completion must not wait for worker startup, parser acceptance, timeout, or recovery. -A future scope-dependent semantic slice will parse once and run adapter-owned -semantic decoders in the worker realm. It will move the closed protocol -atomically to v2 and return a bounded scoped IR, with normalized syntax and -semantic availability represented independently. It will not return flat -relation names, raw ASTs, source text, or absolute document ranges. +The scope decoder runs once in the worker realm immediately after parsing. +It returns query blocks, relation/alias bindings, persistent scopes, typed +visibility regions, independent coverage, and closed issues. It does not +return flat relation names, raw ASTs, source text, or absolute document ranges. +The host revalidates the complete DTO before accepting the response. Worker-local AST caching is deferred until profiling demonstrates that reparsing is material enough to justify leases, byte accounting, generation diff --git a/docs/adr/0006-query-binding-model.md b/docs/adr/0006-query-binding-model.md index 64271af..27eb783 100644 --- a/docs/adr/0006-query-binding-model.md +++ b/docs/adr/0006-query-binding-model.md @@ -97,6 +97,7 @@ expression binding, semantic diagnostics, navigation, rename, formatting, and vendor constructs such as `PIVOT`, `UNNEST`, table functions, and unproven `LATERAL` semantics are explicitly outside this slice. -The next slice is a strict AST-to-plain-data decoder, normalized inside the -worker, with direct-versus-worker differential tests. Public APIs and column -providers follow only after that decoder proves the model. +The node-sql-parser integration now supplies a strict AST-to-plain-data decoder, +normalizes inside the worker, and checks direct-versus-worker parity. Public +semantic APIs remain deferred until a column-completion vertical slice proves +the model against consumer and dialect corpora. diff --git a/src/vnext/__tests__/node-sql-parser-adapter.test.ts b/src/vnext/__tests__/node-sql-parser-adapter.test.ts index b9d61fe..e951f1f 100644 --- a/src/vnext/__tests__/node-sql-parser-adapter.test.ts +++ b/src/vnext/__tests__/node-sql-parser-adapter.test.ts @@ -5,6 +5,7 @@ import { exportedForTesting, getBigQueryNodeSqlStatementParser, getDuckDbCompatibilityNodeSqlStatementParser, + getNodeSqlParserQueryBindingModel, getPostgresqlNodeSqlStatementParser, MAX_NODE_SQL_PARSER_STATEMENT_LENGTH, } from "../node-sql-parser-adapter.js"; @@ -885,6 +886,20 @@ describe("real node-sql-parser builds", () => { mode: "compatibility", status: "parsed", }); + if ( + expectedKind === "query" && + state.analysis.status === "parsed" + ) { + expect( + getNodeSqlParserQueryBindingModel(state.analysis.artifact), + ).toMatchObject({ + coverage: { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }, + }); + } }, ); @@ -935,6 +950,18 @@ describe("real node-sql-parser builds", () => { mode: "compatibility", status: "parsed", }); + if (state.analysis.status === "parsed") { + expect( + getNodeSqlParserQueryBindingModel(state.analysis.artifact), + ).toMatchObject({ + coverage: { + queryBlocks: "partial", + relationBindings: "partial", + visibility: "partial", + }, + issues: [{ code: "parser-compatibility" }], + }); + } }); it("never turns a DuckDB compatibility rejection into invalid SQL", async () => { @@ -953,7 +980,7 @@ describe("real node-sql-parser builds", () => { }); describe("backend artifact privacy", () => { - it("retains backend data only behind the authentic artifact", async () => { + it("retains only normalized bindings behind the authentic artifact", async () => { const backendRoot = { privateBackendField: "must-not-leak", type: "select", @@ -966,7 +993,15 @@ describe("backend artifact privacy", () => { } const { artifact } = state.analysis; - expect(exportedForTesting.hasBackendPayload(artifact)).toBe(true); + expect(exportedForTesting.hasBackendPayload(artifact)).toBe(false); + expect(getNodeSqlParserQueryBindingModel(artifact)).toMatchObject({ + coverage: { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }, + statementRange: { from: 0, to: 8 }, + }); expect(Object.keys(artifact)).toEqual(["kind", "range"]); expect(JSON.stringify(state)).not.toContain("must-not-leak"); expect(Reflect.ownKeys(artifact)).not.toContain("privateBackendField"); @@ -981,6 +1016,12 @@ describe("backend artifact privacy", () => { structuralCopy, ), ).toBe(false); + expect( + invokeWithUnknown( + getNodeSqlParserQueryBindingModel, + structuralCopy, + ), + ).toBeNull(); }); it("does not retain malformed or unsupported backend values", async () => { diff --git a/src/vnext/__tests__/node-sql-parser-browser-executor.test.ts b/src/vnext/__tests__/node-sql-parser-browser-executor.test.ts index 9e86078..20fc812 100644 --- a/src/vnext/__tests__/node-sql-parser-browser-executor.test.ts +++ b/src/vnext/__tests__/node-sql-parser-browser-executor.test.ts @@ -294,7 +294,11 @@ function ready(worker: FakeWorker): void { function respond( worker: FakeWorker, outcome: - | { readonly kind: "parsed"; readonly statementKind: "query" } + | { + readonly kind: "parsed"; + readonly queryBindings: null; + readonly statementKind: "query"; + } | { readonly kind: "syntax-rejected" } | { readonly kind: "unsupported"; @@ -309,8 +313,9 @@ function respond( const backendOutcome = outcome.kind === "parsed" ? { - ...outcome, + kind: outcome.kind, root: Object.freeze({}), + statementKind: outcome.statementKind, } : outcome.kind === "failed" ? { @@ -322,6 +327,7 @@ function respond( encodeNodeSqlParserWireBackendOutcome( postedRequest(worker, index).requestId, backendOutcome, + outcome.kind === "parsed" ? outcome.queryBindings : null, ), ); } @@ -366,7 +372,7 @@ describe("node-sql-parser browser executor admission", () => { expect(request).toStrictEqual({ grammar: "postgresql", kind: "parse", - protocolVersion: 1, + protocolVersion: 2, requestId: 1, text: "SELECT 1", }); @@ -374,11 +380,13 @@ describe("node-sql-parser browser executor admission", () => { respond(worker, { kind: "parsed", + queryBindings: null, statementKind: "query", }); const result = await outcome(submission); expect(result).toStrictEqual({ kind: "parsed", + queryBindings: null, statementKind: "query", }); expect(Object.isFrozen(result)).toBe(true); @@ -429,10 +437,12 @@ describe("node-sql-parser browser executor admission", () => { respond(worker, { kind: "parsed", + queryBindings: null, statementKind: "query", }, 2); expect(await outcome(third)).toStrictEqual({ kind: "parsed", + queryBindings: null, statementKind: "query", }); }); @@ -1210,12 +1220,12 @@ describe("node-sql-parser browser executor hostile worker handling", () => { null, {}, { data: null }, - { data: { kind: "ready", protocolVersion: 2 } }, + { data: { kind: "ready", protocolVersion: 1 } }, { data: { extra: true, kind: "ready", - protocolVersion: 1, + protocolVersion: 2, }, }, ])("fails closed for malformed message event %#", async (event) => { @@ -1285,7 +1295,7 @@ describe("node-sql-parser browser executor hostile worker handling", () => { worker.emit({ code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, } satisfies NodeSqlParserWireMessage); expect(await outcome(submission)).toStrictEqual({ code: "protocol-error", @@ -1311,7 +1321,7 @@ describe("node-sql-parser browser executor hostile worker handling", () => { firstWorker.emit({ kind: "syntax-rejected", - protocolVersion: 1, + protocolVersion: 2, requestId: activeId + 1, } satisfies NodeSqlParserWireMessage); expect(await outcome(active)).toStrictEqual({ diff --git a/src/vnext/__tests__/node-sql-parser-browser-worker-endpoint.test.ts b/src/vnext/__tests__/node-sql-parser-browser-worker-endpoint.test.ts index 749c469..f4a516e 100644 --- a/src/vnext/__tests__/node-sql-parser-browser-worker-endpoint.test.ts +++ b/src/vnext/__tests__/node-sql-parser-browser-worker-endpoint.test.ts @@ -161,7 +161,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.listenerCount()).toBe(1); expect(scope.messages).toStrictEqual([ - { kind: "ready", protocolVersion: 1 }, + { kind: "ready", protocolVersion: 2 }, ]); expect(evaluations).toStrictEqual({ bigquery: 0, @@ -172,7 +172,8 @@ describe("node-sql-parser browser worker endpoint", () => { await waitForMessageCount(scope, 2); expect(scope.messages[1]).toStrictEqual({ kind: "parsed", - protocolVersion: 1, + queryBindings: expect.any(Object), + protocolVersion: 2, requestId: 1, statementKind: "query", }); @@ -185,7 +186,8 @@ describe("node-sql-parser browser worker endpoint", () => { await waitForMessageCount(scope, 3); expect(scope.messages[2]).toStrictEqual({ kind: "parsed", - protocolVersion: 1, + queryBindings: expect.any(Object), + protocolVersion: 2, requestId: 2, statementKind: "insert", }); @@ -330,7 +332,8 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ kind: "parsed", - protocolVersion: 1, + queryBindings: expect.any(Object), + protocolVersion: 2, requestId: 19, statementKind: "query", }); @@ -374,7 +377,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "module-load", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 17, }); expect(JSON.stringify(scope.messages[1])).not.toContain( @@ -420,7 +423,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 1, }); expect(scope.closeCalls()).toBe(1); @@ -457,7 +460,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 8, }); expect(scope.closeCalls()).toBe(1); @@ -508,7 +511,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 10, }); expect(scope.closeCalls()).toBe(1); @@ -547,7 +550,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 11, }); expect(JSON.stringify(scope.messages[1])).not.toContain( @@ -589,7 +592,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 9, }); expect(evaluations).toBe(0); @@ -623,7 +626,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, }); expect(scope.closeCalls()).toBe(1); expect(scope.listenerCount()).toBe(0); @@ -645,11 +648,11 @@ describe("node-sql-parser browser worker endpoint", () => { scope.dispatch({ ...valid, extra: true }); expect(scope.messages).toStrictEqual([ - { kind: "ready", protocolVersion: 1 }, + { kind: "ready", protocolVersion: 2 }, { code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, }, ]); expect(scope.closeCalls()).toBe(1); @@ -678,7 +681,8 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ kind: "parsed", - protocolVersion: 1, + queryBindings: expect.any(Object), + protocolVersion: 2, requestId: 23, statementKind: "query", }); @@ -694,7 +698,7 @@ describe("node-sql-parser browser worker endpoint", () => { code: undefined, expected: { kind: "syntax-rejected", - protocolVersion: 1, + protocolVersion: 2, requestId: 4, }, moduleValue: parserModule(() => { @@ -712,7 +716,7 @@ describe("node-sql-parser browser worker endpoint", () => { code: undefined, expected: { kind: "unsupported", - protocolVersion: 1, + protocolVersion: 2, reason: "multiple-statements", requestId: 4, }, @@ -726,7 +730,7 @@ describe("node-sql-parser browser worker endpoint", () => { expected: { code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 4, }, moduleValue: parserModule(() => { @@ -738,7 +742,7 @@ describe("node-sql-parser browser worker endpoint", () => { expected: { code: "malformed-output", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 4, }, moduleValue: {}, @@ -863,11 +867,11 @@ describe("node-sql-parser browser worker endpoint", () => { scope.dispatch(request(1)); expect(scope.messages).toStrictEqual([ - { kind: "ready", protocolVersion: 1 }, + { kind: "ready", protocolVersion: 2 }, { code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, }, ]); expect(scope.closeCalls()).toBe(1); diff --git a/src/vnext/__tests__/node-sql-parser-query-bindings.bench.ts b/src/vnext/__tests__/node-sql-parser-query-bindings.bench.ts new file mode 100644 index 0000000..ca3c991 --- /dev/null +++ b/src/vnext/__tests__/node-sql-parser-query-bindings.bench.ts @@ -0,0 +1,31 @@ +import { bench, describe } from "vitest"; +import { + normalizeNodeSqlParserQueryBindings, +} from "../node-sql-parser-query-bindings.js"; + +const RELATION_COUNT = 1_000; +const relations = Array.from( + { length: RELATION_COUNT }, + (_, index) => `t${index} a${index}`, +); +const text = `SELECT a999.id FROM ${relations.join(", ")}`; +const root = { + from: Array.from({ length: RELATION_COUNT }, (_, index) => ({ + as: `a${index}`, + db: null, + table: `t${index}`, + })), + type: "select", +}; +const authority = {}; + +describe("node-sql-parser query binding normalization", () => { + bench("normalizes 1,000 relations", () => { + normalizeNodeSqlParserQueryBindings( + root, + text, + authority, + { compatibility: false, grammar: "postgresql" }, + ); + }); +}); diff --git a/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts b/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts new file mode 100644 index 0000000..89f4460 --- /dev/null +++ b/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts @@ -0,0 +1,658 @@ +// @vitest-environment node + +import { describe, expect, it } from "vitest"; +import { + normalizeNodeSqlParserQueryBindings, + type NodeSqlParserQueryBindingOptions, +} from "../node-sql-parser-query-bindings.js"; +import { + resolveSqlRelationQualifier, + visibleSqlRelationBindingsAt, +} from "../query-binding-model.js"; +import { + decodeNodeSqlParserWireMessage, + encodeNodeSqlParserWireBackendOutcome, +} from "../node-sql-parser-wire.js"; +import type { SqlIdentifierComponent } from "../types.js"; + +const PG: NodeSqlParserQueryBindingOptions = { + compatibility: false, + grammar: "postgresql", +}; +const BQ: NodeSqlParserQueryBindingOptions = { + compatibility: false, + grammar: "bigquery", +}; + +function normalize( + root: unknown, + text: string, + options: NodeSqlParserQueryBindingOptions = PG, +) { + return normalizeNodeSqlParserQueryBindings(root, text, {}, options); +} + +function relation(table: string, alias: string | null = null) { + return { as: alias, db: null, table }; +} + +function equals( + left: SqlIdentifierComponent, + right: SqlIdentifierComponent, +): boolean { + return left.quoted === right.quoted && + (left.quoted + ? left.value === right.value + : left.value.toLowerCase() === right.value.toLowerCase()); +} + +describe("node-sql-parser query binding normalization", () => { + it("normalizes named relations, aliases, scopes, and clause visibility", () => { + const text = + "SELECT u.id FROM users AS u JOIN teams t ON u.team_id=t.id " + + "WHERE u.active GROUP BY u.id HAVING count(*)>1 " + + "ORDER BY u.id LIMIT 5"; + const result = normalize( + { + from: [ + relation("users", "u"), + { ...relation("teams", "t"), join: "JOIN", on: {} }, + ], + type: "select", + where: {}, + }, + text, + ); + + expect(result.status).toBe("ready"); + if (result.status !== "ready") { + return; + } + expect(result.model.coverage).toEqual({ + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }); + expect(result.model.bindings).toMatchObject([ + { + alias: { explicit: true, name: { value: "u" } }, + source: { kind: "named", path: [{ value: "users" }] }, + }, + { + alias: { explicit: false, name: { value: "t" } }, + source: { kind: "named", path: [{ value: "teams" }] }, + }, + ]); + expect(result.model.regions.map((region) => region.kind)).toEqual( + expect.arrayContaining([ + "select-list", + "from-source", + "join-condition", + "where", + "group-by", + "having", + "order-by", + "limit", + ]), + ); + const selected = visibleSqlRelationBindingsAt( + result.model, + text.indexOf("u.id"), + ); + expect(selected.status === "ready" && selected.bindings).toHaveLength(2); + expect( + resolveSqlRelationQualifier( + result.model, + text.indexOf("u.id"), + { quoted: false, value: "U" }, + equals, + ).status, + ).toBe("resolved"); + }); + + it("normalizes derived query blocks without leaking sibling scope", () => { + const text = + "SELECT d.id FROM (SELECT id FROM teams) AS d " + + "JOIN users u ON d.id=u.id"; + const derived = { + from: [relation("teams")], + type: "select", + }; + const result = normalize( + { + from: [ + { + as: "d", + expr: { ast: derived, parentheses: true }, + }, + { ...relation("users", "u"), join: "JOIN", on: {} }, + ], + type: "select", + }, + text, + ); + + expect(result.status).toBe("ready"); + if (result.status !== "ready") { + return; + } + expect(result.model.blocks).toHaveLength(2); + expect(result.model.blocks[1]?.parentBlock).toBe(0); + expect(result.model.bindings).toMatchObject([ + { alias: { name: { value: "d" } }, source: { block: 1, kind: "derived" } }, + { alias: { name: { value: "u" } }, source: { kind: "named" } }, + { owner: 1, source: { kind: "named", path: [{ value: "teams" }] } }, + ]); + expect(result.model.blocks[1]?.baseScope).toBe(0); + }); + + it("resolves CTE references to declaration evidence", () => { + const text = + "WITH recent AS (SELECT * FROM events) " + + "SELECT * FROM recent r"; + const cteStatement = { + from: [relation("events")], + type: "select", + }; + const result = normalize( + { + from: [relation("recent", "r")], + type: "select", + with: [ + { + name: { type: "default", value: "recent" }, + stmt: cteStatement, + }, + ], + }, + text, + ); + + expect(result.status).toBe("ready"); + if (result.status === "ready") { + expect(result.model.blocks).toHaveLength(2); + expect(result.model.bindings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + source: expect.objectContaining({ + kind: "cte", + name: { quoted: false, value: "recent" }, + }), + }), + ]), + ); + } + }); + + it("normalizes BigQuery multipart paths and QUALIFY", () => { + const text = + "SELECT a.id FROM `project.dataset.users` a " + + "QUALIFY row_number() over()=1"; + const result = normalize( + { + from: [ + { + as: "a", + db: null, + surround: { table: "`" }, + table: "project.dataset.users", + }, + ], + qualify: {}, + type: "select", + }, + text, + BQ, + ); + + expect(result.status).toBe("ready"); + if (result.status === "ready") { + expect(result.model.bindings[0]?.source).toMatchObject({ + kind: "named", + path: [ + { value: "project" }, + { value: "dataset" }, + { value: "users" }, + ], + }); + expect( + result.model.regions.some((region) => region.kind === "qualify"), + ).toBe(true); + } + }); + + it("marks DuckDB compatibility evidence partial in every dimension", () => { + const result = normalize( + { from: [relation("events")], type: "select" }, + "SELECT * FROM events", + { compatibility: true, grammar: "postgresql" }, + ); + expect(result).toMatchObject({ + model: { + coverage: { + queryBlocks: "partial", + relationBindings: "partial", + visibility: "partial", + }, + issues: [{ code: "parser-compatibility" }], + }, + status: "ready", + }); + }); + + it("downgrades unmatched lexical or parser relations without guessing", () => { + const result = normalize( + { + from: [ + relation("users"), + { as: null, expr: { type: "function" } }, + ], + type: "select", + }, + "SELECT * FROM users", + ); + expect(result).toMatchObject({ + model: { + coverage: { + relationBindings: "partial", + visibility: "partial", + }, + issues: [{ code: "unsupported-relation-source" }], + }, + status: "ready", + }); + }); + + it("marks extra lexical query blocks and unknown relation shapes partial", () => { + const extraBlock = normalize( + { type: "select" }, + "SELECT EXISTS (SELECT 1)", + ); + expect(extraBlock).toMatchObject({ + model: { + coverage: { queryBlocks: "partial" }, + issues: [{ code: "unsupported-clause" }], + }, + status: "ready", + }); + + const unknownRelation = normalize( + { from: [{}], type: "select" }, + "SELECT * FROM users", + ); + expect(unknownRelation).toMatchObject({ + model: { + coverage: { relationBindings: "partial" }, + bindings: [ + { + source: { + kind: "unknown", + reason: "unsupported-relation-source", + }, + }, + ], + }, + status: "ready", + }); + }); + + it("downgrades alias disagreement instead of trusting either side", () => { + const lexicalOnly = normalize( + { from: [relation("users")], type: "select" }, + "SELECT * FROM users u", + ); + const astOnly = normalize( + { from: [relation("users", "u")], type: "select" }, + "SELECT * FROM users", + ); + for (const result of [lexicalOnly, astOnly]) { + expect(result).toMatchObject({ + model: { + coverage: { relationBindings: "partial" }, + issues: [{ code: "unsupported-relation-source" }], + }, + status: "ready", + }); + } + }); + + it("preserves PostgreSQL quoted path components", () => { + const result = normalize( + { + from: [{ as: "U", db: "MySchema", table: "Users" }], + type: "select", + }, + 'SELECT * FROM "MySchema"."Users" AS "U"', + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { + alias: { name: { quoted: true, value: "U" } }, + source: { + kind: "named", + path: [ + { quoted: true, value: "MySchema" }, + { quoted: true, value: "Users" }, + ], + }, + }, + ], + }, + status: "ready", + }); + }); + + it("round trips normalized data across wire v2 without raw AST or text", () => { + const text = "SELECT u.id FROM users u"; + const result = normalize( + { from: [relation("users", "u")], type: "select" }, + text, + ); + if (result.status !== "ready") { + throw new Error("Expected normalized query bindings"); + } + const root = { privateAstSecret: "never-cross-wire", type: "select" }; + const encoded = encodeNodeSqlParserWireBackendOutcome( + 1, + { kind: "parsed", root, statementKind: "query" }, + result.model, + ); + const decoded = decodeNodeSqlParserWireMessage(encoded); + + expect(decoded).not.toBeNull(); + expect(decoded?.kind).toBe("parsed"); + if (decoded?.kind === "parsed") { + expect(decoded.queryBindings).toStrictEqual(result.model); + expect(decoded.queryBindings).not.toBe(result.model); + } + expect(JSON.stringify(encoded)).not.toContain("privateAstSecret"); + expect(JSON.stringify(encoded)).not.toContain(text); + expect(Object.hasOwn(encoded, "root")).toBe(false); + }); +}); + +describe("node-sql-parser query binding hostile boundaries", () => { + it("does not invoke AST accessors", () => { + let calls = 0; + const root = {}; + Object.defineProperty(root, "type", { + get() { + calls += 1; + return "select"; + }, + }); + expect(normalize(root, "SELECT 1")).toEqual({ + reason: "not-query", + status: "unavailable", + }); + expect(calls).toBe(0); + }); + + it("rejects cycles and sparse or oversized AST arrays", () => { + const cyclic = { from: [] as unknown[], type: "select" }; + cyclic.from.push({ expr: { ast: cyclic } }); + expect(normalize(cyclic, "SELECT * FROM (SELECT 1) x")).toEqual({ + reason: "malformed-ast", + status: "unavailable", + }); + + const sparse: unknown[] = []; + sparse.length = 1; + expect(normalize({ from: sparse, type: "select" }, "SELECT 1")).toEqual({ + reason: "malformed-ast", + status: "unavailable", + }); + + const oversized = Array.from({ length: 1_025 }, () => relation("x")); + expect(normalize({ from: oversized, type: "select" }, "SELECT 1")).toEqual({ + reason: "malformed-ast", + status: "unavailable", + }); + }); + + it("rejects malformed known AST properties and CTE entries", () => { + const cases: readonly { + readonly root: unknown; + readonly text: string; + }[] = [ + { root: { from: {}, type: "select" }, text: "SELECT 1" }, + { root: { from: [null], type: "select" }, text: "SELECT 1" }, + { root: { type: "select", with: {} }, text: "SELECT 1" }, + { root: { type: "select", with: [null] }, text: "SELECT 1" }, + { + root: { type: "select", with: [{}] }, + text: "WITH x AS (SELECT 1) SELECT 1", + }, + { + root: { + type: "select", + with: [{ name: {}, stmt: { type: "select" } }], + }, + text: "WITH x AS (SELECT 1) SELECT 1", + }, + { + root: { + type: "select", + with: [{ name: "missing", stmt: { type: "select" } }], + }, + text: "WITH present AS (SELECT 1) SELECT 1", + }, + ]; + for (const { root, text } of cases) { + expect(normalize(root, text).status).toBe("unavailable"); + } + }); + + it("handles comments, incomplete relation sites, and nesting limits", () => { + expect( + normalize( + { from: [relation("users")], type: "select" }, + "SELECT /* private */ * FROM users -- tail", + ).status, + ).toBe("ready"); + expect( + normalize({ type: "select" }, "SELECT * FROM").status, + ).toBe("ready"); + expect( + normalize( + { type: "select" }, + `${"(".repeat(257)}SELECT 1${")".repeat(257)}`, + ), + ).toEqual({ reason: "resource-limit", status: "unavailable" }); + }); + + it("normalizes descriptor traps and invalid normalization authority", () => { + const hostile = new Proxy( + { type: "select" }, + { + getOwnPropertyDescriptor() { + throw new Error("private trap"); + }, + }, + ); + expect(normalize(hostile, "SELECT 1")).toEqual({ + reason: "not-query", + status: "unavailable", + }); + expect( + normalizeNodeSqlParserQueryBindings( + { type: "select" }, + "SELECT 1", + null, + PG, + ), + ).toEqual({ reason: "resource-limit", status: "unavailable" }); + + const fromAccessor = { type: "select" }; + Object.defineProperty(fromAccessor, "from", { + get() { + throw new Error("private from getter"); + }, + }); + expect(normalize(fromAccessor, "SELECT 1")).toEqual({ + reason: "malformed-ast", + status: "unavailable", + }); + expect( + normalize({ from: null, type: "select" }, "SELECT 1").status, + ).toBe("ready"); + }); + + it("bounds recursive CTEs, unclosed derived sites, and identifier paths", () => { + const recursive = { + type: "select", + with: [] as unknown[], + }; + recursive.with.push({ name: "self", stmt: recursive }); + expect( + normalize( + recursive, + "WITH self AS (SELECT 1) SELECT 1", + ), + ).toEqual({ reason: "malformed-ast", status: "unavailable" }); + + const child = { type: "select" }; + expect( + normalize( + { + from: [{ as: "d", expr: { ast: child } }], + type: "select", + }, + "SELECT * FROM (SELECT 1", + ), + ).toMatchObject({ + model: { coverage: { relationBindings: "partial" } }, + status: "ready", + }); + + const longName = "x".repeat(257); + expect( + normalize( + { from: [relation(longName)], type: "select" }, + `SELECT * FROM ${longName}`, + ), + ).toMatchObject({ + model: { + bindings: [{ source: { kind: "unknown" } }], + coverage: { relationBindings: "partial" }, + }, + status: "ready", + }); + + const manyComponents = Array.from( + { length: 9 }, + (_, index) => `p${index}`, + ).join("."); + expect( + normalize( + { from: [relation(manyComponents)], type: "select" }, + `SELECT * FROM \`${manyComponents}\``, + BQ, + ), + ).toMatchObject({ + model: { bindings: [{ source: { kind: "unknown" } }] }, + status: "ready", + }); + }); + + it("covers alternate bounded grammar shapes without widening evidence", () => { + expect(normalize({ type: "union" }, "SELECT 1").status).toBe("ready"); + expect( + normalize( + { + from: [{ as: null, schema: "public", table: "users" }], + type: "select", + }, + "SELECT * FROM public.users", + ), + ).toMatchObject({ + model: { + bindings: [ + { + source: { + path: [{ value: "public" }, { value: "users" }], + }, + }, + ], + }, + status: "ready", + }); + expect( + normalize( + { + type: "select", + with: [{ name: "x", stmt: { type: "select" } }], + }, + "WITH x AS (SELECT 1) SELECT 1", + ).status, + ).toBe("ready"); + expect( + normalize( + { from: [relation("users")], type: "select" }, + 'SELECT * FROM ""', + ).status, + ).toBe("ready"); + expect(normalize({ type: "select" }, "SELECT 1 ON true").status).toBe( + "ready", + ); + expect( + normalize( + { type: "select" }, + `SELECT $${"a".repeat(300)}$`, + ), + ).toEqual({ reason: "resource-limit", status: "unavailable" }); + expect( + normalizeNodeSqlParserQueryBindings( + { type: "select" }, + 1, + {}, + PG, + ), + ).toEqual({ reason: "resource-limit", status: "unavailable" }); + }); + + it("rejects malformed roots, syntax shapes, and resource excess", () => { + expect(normalize(null, "SELECT 1")).toEqual({ + reason: "not-query", + status: "unavailable", + }); + expect(normalize({ type: "insert" }, "INSERT INTO x VALUES (1)")).toEqual({ + reason: "not-query", + status: "unavailable", + }); + expect(normalize({ type: "select" }, "x".repeat(16 * 1024 + 1))).toEqual({ + reason: "resource-limit", + status: "unavailable", + }); + expect(normalize({ type: "select" }, "SELECT 1)")).toEqual({ + reason: "resource-limit", + status: "unavailable", + }); + expect(normalize({ type: "select" }, "not a query")).toEqual({ + reason: "unsupported-shape", + status: "unavailable", + }); + }); + + it("rejects hostile wire binding payloads without invoking getters", () => { + let calls = 0; + const hostile = {}; + Object.defineProperty(hostile, "statementRange", { + enumerable: true, + get() { + calls += 1; + return { from: 0, to: 8 }; + }, + }); + const decoded = decodeNodeSqlParserWireMessage({ + kind: "parsed", + protocolVersion: 2, + queryBindings: hostile, + requestId: 1, + statementKind: "query", + }); + expect(decoded).toBeNull(); + expect(calls).toBe(0); + }); +}); diff --git a/src/vnext/__tests__/node-sql-parser-wire.test.ts b/src/vnext/__tests__/node-sql-parser-wire.test.ts index 7647ca5..11a5eb3 100644 --- a/src/vnext/__tests__/node-sql-parser-wire.test.ts +++ b/src/vnext/__tests__/node-sql-parser-wire.test.ts @@ -47,6 +47,7 @@ const validMessages: readonly NodeSqlParserWireMessage[] = [ ...statementKinds.map( (statementKind): NodeSqlParserWireMessage => ({ kind: "parsed", + queryBindings: null, protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, requestId: 1, statementKind, @@ -207,7 +208,7 @@ describe("node-sql-parser wire request codec", () => { expect(encoded).toStrictEqual({ grammar, kind: "parse", - protocolVersion: 1, + protocolVersion: 2, requestId: Number.MAX_SAFE_INTEGER, text: " SELECT 1\n", }); @@ -289,7 +290,7 @@ describe("node-sql-parser wire request codec", () => { ).toBeNull(); }); - it.each([0, 2, "1", null])( + it.each([0, 1, 3, "2", null])( "rejects protocol version %#", (protocolVersion) => { expect( @@ -376,7 +377,8 @@ describe("node-sql-parser wire message codec", () => { encodeNodeSqlParserWireBackendOutcome(7, outcome), ).toStrictEqual({ kind: "parsed", - protocolVersion: 1, + queryBindings: null, + protocolVersion: 2, requestId: 7, statementKind, }); @@ -392,7 +394,7 @@ describe("node-sql-parser wire message codec", () => { { kind: "syntax-rejected" }, { kind: "syntax-rejected", - protocolVersion: 1, + protocolVersion: 2, requestId: 3, }, ], @@ -400,7 +402,7 @@ describe("node-sql-parser wire message codec", () => { { kind: "unsupported", reason: "multiple-statements" }, { kind: "unsupported", - protocolVersion: 1, + protocolVersion: 2, reason: "multiple-statements", requestId: 3, }, @@ -409,7 +411,7 @@ describe("node-sql-parser wire message codec", () => { { kind: "unsupported", reason: "resource-limit" }, { kind: "unsupported", - protocolVersion: 1, + protocolVersion: 2, reason: "resource-limit", requestId: 3, }, @@ -419,7 +421,7 @@ describe("node-sql-parser wire message codec", () => { { code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 3, }, ], @@ -432,7 +434,7 @@ describe("node-sql-parser wire message codec", () => { { code: "malformed-output", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 3, }, ], @@ -441,7 +443,7 @@ describe("node-sql-parser wire message codec", () => { { code: "module-load", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 3, }, ], @@ -450,7 +452,7 @@ describe("node-sql-parser wire message codec", () => { const ready = encodeNodeSqlParserWireReady(); expect(ready).toStrictEqual({ kind: "ready", - protocolVersion: 1, + protocolVersion: 2, }); expect(Object.isFrozen(ready)).toBe(true); for (const [outcome, expected] of cases) { @@ -493,6 +495,7 @@ describe("node-sql-parser wire message codec", () => { expect(Reflect.ownKeys(parsed)).toStrictEqual([ "kind", "protocolVersion", + "queryBindings", "requestId", "statementKind", ]); @@ -532,7 +535,7 @@ describe("node-sql-parser wire message codec", () => { expect(protocolError).toStrictEqual({ code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, }); expect(Reflect.ownKeys(protocolError)).not.toContain("requestId"); expect(Object.isFrozen(protocolError)).toBe(true); @@ -568,7 +571,7 @@ describe("node-sql-parser wire message codec", () => { expect( decodeNodeSqlParserWireMessage({ kind: "syntax-rejected", - protocolVersion: 1, + protocolVersion: 2, requestId, }), ).toBeNull(); @@ -584,7 +587,8 @@ describe("node-sql-parser wire message codec", () => { expect( decodeNodeSqlParserWireMessage({ kind: "parsed", - protocolVersion: 1, + queryBindings: null, + protocolVersion: 2, requestId: 1, statementKind, }), @@ -600,7 +604,7 @@ describe("node-sql-parser wire message codec", () => { expect( decodeNodeSqlParserWireMessage({ kind: "unsupported", - protocolVersion: 1, + protocolVersion: 2, reason, requestId: 1, }), @@ -617,7 +621,7 @@ describe("node-sql-parser wire message codec", () => { decodeNodeSqlParserWireMessage({ code, kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 1, }), ).toBeNull(); @@ -632,12 +636,12 @@ describe("node-sql-parser wire message codec", () => { expect( decodeNodeSqlParserWireMessage({ kind, - protocolVersion: 1, + protocolVersion: 2, }), ).toBeNull(); }); - it.each([0, 2, "1", null])( + it.each([0, 1, 3, "2", null])( "rejects response protocol version %#", (protocolVersion) => { expect( @@ -654,7 +658,7 @@ describe("node-sql-parser wire message codec", () => { decodeNodeSqlParserWireMessage({ code: "module-load", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 1, retryable: true, }), @@ -663,7 +667,7 @@ describe("node-sql-parser wire message codec", () => { decodeNodeSqlParserWireMessage({ code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, requestId: 1, }), ).toBeNull(); diff --git a/src/vnext/browser_tests/fixtures/node-sql-parser-crash-worker.js b/src/vnext/browser_tests/fixtures/node-sql-parser-crash-worker.js index 530f6a4..1ec2a4f 100644 --- a/src/vnext/browser_tests/fixtures/node-sql-parser-crash-worker.js +++ b/src/vnext/browser_tests/fixtures/node-sql-parser-crash-worker.js @@ -1,6 +1,6 @@ globalThis.postMessage({ kind: "ready", - protocolVersion: 1, + protocolVersion: 2, }); globalThis.addEventListener("message", () => { throw new Error("Intentional parser executor crash fixture"); diff --git a/src/vnext/browser_tests/fixtures/node-sql-parser-silent-worker.js b/src/vnext/browser_tests/fixtures/node-sql-parser-silent-worker.js index b2c2edd..7ae4382 100644 --- a/src/vnext/browser_tests/fixtures/node-sql-parser-silent-worker.js +++ b/src/vnext/browser_tests/fixtures/node-sql-parser-silent-worker.js @@ -1,5 +1,5 @@ globalThis.postMessage({ kind: "ready", - protocolVersion: 1, + protocolVersion: 2, }); globalThis.addEventListener("message", () => {}); diff --git a/src/vnext/browser_tests/node-sql-parser-browser-executor.test.ts b/src/vnext/browser_tests/node-sql-parser-browser-executor.test.ts index 08a6b9b..eac837f 100644 --- a/src/vnext/browser_tests/node-sql-parser-browser-executor.test.ts +++ b/src/vnext/browser_tests/node-sql-parser-browser-executor.test.ts @@ -177,6 +177,7 @@ test( ), ).resolves.toStrictEqual({ kind: "parsed", + queryBindings: expect.any(Object), statementKind: "query", }); await expect( @@ -187,6 +188,7 @@ test( ), ).resolves.toStrictEqual({ kind: "parsed", + queryBindings: expect.any(Object), statementKind: "query", }); await expect( @@ -197,6 +199,7 @@ test( ), ).resolves.toStrictEqual({ kind: "parsed", + queryBindings: expect.any(Object), statementKind: "query", }); } finally { @@ -232,6 +235,7 @@ test( submitQuery(executor, "postgresql", "SELECT 2"), ).resolves.toStrictEqual({ kind: "parsed", + queryBindings: expect.any(Object), statementKind: "query", }); expect(generation).toBe(2); @@ -268,6 +272,7 @@ test( submitQuery(executor, "bigquery", "SELECT 2"), ).resolves.toStrictEqual({ kind: "parsed", + queryBindings: expect.any(Object), statementKind: "query", }); expect(generation).toBe(2); diff --git a/src/vnext/node-sql-parser-adapter.ts b/src/vnext/node-sql-parser-adapter.ts index 86d80d6..197aa4f 100644 --- a/src/vnext/node-sql-parser-adapter.ts +++ b/src/vnext/node-sql-parser-adapter.ts @@ -5,6 +5,11 @@ import { type NodeSqlParserModuleLoadOutcome, type NodeSqlParserModuleLoader, } from "./node-sql-parser-backend.js"; +import { + normalizeNodeSqlParserQueryBindings, + type NodeSqlParserQueryBindingGrammar, +} from "./node-sql-parser-query-bindings.js"; +import type { SqlQueryBindingModel } from "./query-binding-model.js"; import { createCompatibilityParsedAnalysis, createFailedParserAnalysis, @@ -39,6 +44,7 @@ type NodeSqlParserPolicy = interface NodeSqlParserRuntime { readonly authority: SqlParserAuthority; readonly backend: NodeSqlParserBackend; + readonly grammar: NodeSqlParserQueryBindingGrammar; readonly limitations: readonly [ SqlCompatibilityLimitation, ...SqlCompatibilityLimitation[], @@ -61,7 +67,8 @@ type OwnDataProperty = class NodeSqlParserGlobalCleanupError extends Error {} class NodeSqlParserExecutionRealmError extends Error {} -const backendPayloads = new WeakMap(); +const queryBindingModels = + new WeakMap(); function readOwnDataProperty( value: object, @@ -307,7 +314,18 @@ function materializeBackendOutcome( statementText, runtime.authority, ); - backendPayloads.set(artifact, outcome.root); + const bindings = normalizeNodeSqlParserQueryBindings( + outcome.root, + statementText, + runtime.authority, + { + compatibility: runtime.policy === "dialect-compatibility", + grammar: runtime.grammar, + }, + ); + if (bindings.status === "ready") { + queryBindingModels.set(artifact, bindings.model); + } return createCompatibilityParsedAnalysis( artifact, runtime.limitations, @@ -349,6 +367,7 @@ function createAdapter(runtime: NodeSqlParserRuntime): SqlStatementParser { function createRuntime( backendIdentity: SqlSyntaxBackendIdentity, backend: NodeSqlParserBackend, + grammar: NodeSqlParserQueryBindingGrammar, policy: NodeSqlParserPolicy, ): NodeSqlParserRuntime { const authority = createSqlParserAuthority( @@ -359,6 +378,7 @@ function createRuntime( return Object.freeze({ authority, backend, + grammar, limitations: policy === "dialect-compatibility" ? DIALECT_COMPATIBILITY_LIMITATIONS @@ -378,6 +398,7 @@ const postgresqlParser = createAdapter( createRuntime( postgresqlBackendIdentity, postgresqlBackend, + "postgresql", "target-grammar", ), ); @@ -385,6 +406,7 @@ const bigQueryParser = createAdapter( createRuntime( bigQueryBackendIdentity, bigQueryBackend, + "bigquery", "target-grammar", ), ); @@ -392,6 +414,7 @@ const duckDbParser = createAdapter( createRuntime( postgresqlBackendIdentity, postgresqlBackend, + "postgresql", "dialect-compatibility", ), ); @@ -449,12 +472,19 @@ export const exportedForTesting = Object.freeze({ createNodeSqlParserBackend( adaptModuleLoaderForTesting(loadModule), ), + "postgresql", policy, ), ); }, - hasBackendPayload(artifact: SqlSyntaxArtifact): boolean { - return backendPayloads.has(artifact); + hasBackendPayload(_artifact: SqlSyntaxArtifact): boolean { + return false; }, createSynchronousModuleLoader, }); + +export function getNodeSqlParserQueryBindingModel( + artifact: SqlSyntaxArtifact, +): SqlQueryBindingModel | null { + return queryBindingModels.get(artifact) ?? null; +} diff --git a/src/vnext/node-sql-parser-browser-executor.ts b/src/vnext/node-sql-parser-browser-executor.ts index f250ec6..0c02e35 100644 --- a/src/vnext/node-sql-parser-browser-executor.ts +++ b/src/vnext/node-sql-parser-browser-executor.ts @@ -7,6 +7,7 @@ import { type NodeSqlParserWireFailureCode, type NodeSqlParserWireGrammar, } from "./node-sql-parser-wire.js"; +import type { SqlQueryBindingModelData } from "./query-binding-model.js"; import type { SqlStatementKind } from "./syntax.js"; export interface NodeSqlParserBrowserExecutorLimits { @@ -68,6 +69,7 @@ export type NodeSqlParserBrowserExecutorFailureCode = export type NodeSqlParserBrowserExecutorOutcome = | { readonly kind: "parsed"; + readonly queryBindings: SqlQueryBindingModelData | null; readonly statementKind: SqlStatementKind; } | { @@ -836,6 +838,7 @@ export function createNodeSqlParserBrowserExecutor( request, Object.freeze({ kind: "parsed", + queryBindings: message.queryBindings, statementKind: message.statementKind, }), ); diff --git a/src/vnext/node-sql-parser-browser-worker-endpoint.ts b/src/vnext/node-sql-parser-browser-worker-endpoint.ts index 7023fbe..8af377f 100644 --- a/src/vnext/node-sql-parser-browser-worker-endpoint.ts +++ b/src/vnext/node-sql-parser-browser-worker-endpoint.ts @@ -3,6 +3,10 @@ import { type NodeSqlParserBackendOutcome, type NodeSqlParserModuleLoadOutcome, } from "./node-sql-parser-backend.js"; +import { + normalizeNodeSqlParserQueryBindings, +} from "./node-sql-parser-query-bindings.js"; +import type { SqlQueryBindingModel } from "./query-binding-model.js"; import { decodeNodeSqlParserWireRequest, encodeNodeSqlParserWireBackendOutcome, @@ -250,6 +254,10 @@ export function installNodeSqlParserBrowserWorkerEndpoint( createModuleLoader(loaders.postgresql), ), } satisfies Record); + const bindingAuthorities = Object.freeze({ + bigquery: Object.freeze({}), + postgresql: Object.freeze({}), + } satisfies Record); function closeEndpoint(): void { state = "closed"; @@ -293,11 +301,27 @@ export function installNodeSqlParserBrowserWorkerEndpoint( guard, ); state = closeAfterSettlement ? "closed" : "idle"; + let queryBindings: SqlQueryBindingModel | null = null; + if (outcome.kind === "parsed") { + const normalized = normalizeNodeSqlParserQueryBindings( + outcome.root, + request.text, + bindingAuthorities[request.grammar], + { + compatibility: false, + grammar: request.grammar, + }, + ); + if (normalized.status === "ready") { + queryBindings = normalized.model; + } + } try { scope.postMessage( encodeNodeSqlParserWireBackendOutcome( request.requestId, outcome, + queryBindings, ), ); } catch { diff --git a/src/vnext/node-sql-parser-query-bindings.ts b/src/vnext/node-sql-parser-query-bindings.ts new file mode 100644 index 0000000..e106c8e --- /dev/null +++ b/src/vnext/node-sql-parser-query-bindings.ts @@ -0,0 +1,984 @@ +import { + BoundedSqlLexer, + type BoundedSqlLexeme, +} from "./bounded-sql-lexer.js"; +import { + BIGQUERY_SQL_LEXICAL_PROFILE, + POSTGRESQL_SQL_LEXICAL_PROFILE, +} from "./lexical.js"; +import { + createSqlQueryBindingModel, + MAX_QUERY_BINDING_BLOCKS, + MAX_QUERY_BINDINGS, + type SqlQueryBindingIssue, + type SqlQueryBindingModel, + type SqlRelationBinding, + type SqlRelationScope, + type SqlVisibilityRegion, +} from "./query-binding-model.js"; +import { createIdentitySqlSource } from "./source.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, + SqlTextRange, +} from "./types.js"; + +export type NodeSqlParserQueryBindingGrammar = + | "bigquery" + | "postgresql"; + +export interface NodeSqlParserQueryBindingOptions { + readonly compatibility: boolean; + readonly grammar: NodeSqlParserQueryBindingGrammar; +} + +export type NodeSqlParserQueryBindingResult = + | { + readonly model: SqlQueryBindingModel; + readonly status: "ready"; + } + | { + readonly reason: + | "malformed-ast" + | "not-query" + | "resource-limit" + | "unsupported-shape"; + readonly status: "unavailable"; + }; + +interface Token extends BoundedSqlLexeme { + readonly depth: number; + readonly text: string; +} + +interface SelectSkeleton { + readonly depth: number; + readonly from: number; + readonly selectToken: number; + readonly to: number; +} + +interface AstSelect { + readonly children: AstSelect[]; + readonly node: object; + skeleton: SelectSkeleton | null; +} + +interface RelationSite { + readonly aliasToken: Token | null; + readonly derived: boolean; + readonly explicitAlias: boolean; + readonly from: number; + readonly lexicalIdentifiers: readonly SqlIdentifierComponent[]; + readonly to: number; +} + +interface BlockBuild { + readonly ast: AstSelect; + readonly id: number; + readonly parent: number | null; + readonly range: SqlTextRange; + readonly skeleton: SelectSkeleton; +} + +interface MutableCoverage { + queryBlocks: "complete" | "partial"; + relationBindings: "complete" | "partial"; + visibility: "complete" | "partial"; +} + +function phaseValue( + values: readonly Value[], + index: number, +): Value { + const value = values[index]; + if (value === undefined) { + throw new Error("Query binding normalization phase invariant failed"); + } + return value; +} + +function assignedSkeleton(ast: AstSelect): SelectSkeleton { + if (ast.skeleton === null) { + throw new Error("Query binding skeleton assignment invariant failed"); + } + return ast.skeleton; +} + +type OwnProperty = + | { readonly kind: "invalid" | "missing" } + | { readonly kind: "value"; readonly value: unknown }; + +const MAX_AST_ARRAY_LENGTH = 1_024; +const MAX_AST_PROPERTY_TEXT = 2_048; + +function ownProperty(value: object, key: PropertyKey): OwnProperty { + try { + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (descriptor === undefined) { + return { kind: "missing" }; + } + if (!("value" in descriptor)) { + return { kind: "invalid" }; + } + return { kind: "value", value: descriptor.value }; + } catch { + return { kind: "invalid" }; + } +} + +function objectProperty(value: object, key: PropertyKey): object | null { + const property = ownProperty(value, key); + return property.kind === "value" && + property.value !== null && + typeof property.value === "object" + ? property.value + : null; +} + +function stringProperty(value: object, key: PropertyKey): string | null { + const property = ownProperty(value, key); + return property.kind === "value" && + typeof property.value === "string" && + property.value.length > 0 && + property.value.length <= MAX_AST_PROPERTY_TEXT + ? property.value + : null; +} + +function arrayProperty( + value: object, + key: PropertyKey, +): readonly unknown[] | null { + const property = ownProperty(value, key); + if (property.kind === "missing") { + return []; + } + if (property.kind !== "value") { + return null; + } + if (property.value === null) { + return []; + } + if (!Array.isArray(property.value)) { + return null; + } + const length = ownProperty(property.value, "length"); + if ( + length.kind !== "value" || + typeof length.value !== "number" || + !Number.isSafeInteger(length.value) || + length.value < 0 || + length.value > MAX_AST_ARRAY_LENGTH + ) { + return null; + } + const items: unknown[] = []; + for (let index = 0; index < length.value; index += 1) { + const item = ownProperty(property.value, index); + if (item.kind !== "value") { + return null; + } + items.push(item.value); + } + return items; +} + +function word(token: Token | undefined, expected: string): boolean { + return ( + token?.kind === "word" && + token.text.toLowerCase() === expected + ); +} + +function tokenize( + text: string, + grammar: NodeSqlParserQueryBindingGrammar, +): readonly Token[] | null { + const source = createIdentitySqlSource(text); + const lexer = new BoundedSqlLexer( + source, + 0, + text.length, + grammar === "bigquery" + ? BIGQUERY_SQL_LEXICAL_PROFILE + : POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + const tokens: Token[] = []; + let depth = 0; + for (;;) { + const lexeme = lexer.next(); + if (lexeme === null) { + return lexer.resource === null ? tokens : null; + } + if ( + lexeme.kind === "comment" || + lexeme.kind === "line-comment" + ) { + continue; + } + const raw = text.slice(lexeme.from, lexeme.to); + if (raw === ")") { + depth -= 1; + if (depth < 0) { + return null; + } + } + tokens.push(Object.freeze({ + ...lexeme, + depth, + text: raw, + })); + if (raw === "(") { + depth += 1; + if (depth > MAX_QUERY_BINDING_BLOCKS) { + return null; + } + } + } +} + +function selectSkeletons( + tokens: readonly Token[], + statementLength: number, +): readonly SelectSkeleton[] { + const skeletons: SelectSkeleton[] = []; + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]; + if (!token || !word(token, "select")) { + continue; + } + let to = statementLength; + for (let cursor = index + 1; cursor < tokens.length; cursor += 1) { + const candidate = tokens[cursor]; + if ( + candidate?.text === ")" && + candidate.depth < token.depth + ) { + to = candidate.from; + break; + } + } + skeletons.push(Object.freeze({ + depth: token.depth, + from: token.from, + selectToken: index, + to, + })); + } + return skeletons; +} + +function astSelectNode(value: unknown): object | null { + if (value === null || typeof value !== "object") { + return null; + } + const type = stringProperty(value, "type"); + return type !== null && + (type.toLowerCase() === "select" || + type.toLowerCase() === "union") + ? value + : null; +} + +function derivedAst(value: object): object | null { + const expression = objectProperty(value, "expr"); + if (expression === null) { + return null; + } + return astSelectNode(objectProperty(expression, "ast")); +} + +function buildAstTree( + root: object, +): { readonly root: AstSelect; readonly textual: readonly AstSelect[] } | null { + const seen = new WeakSet(); + const textual: AstSelect[] = []; + let count = 0; + + function visit(node: object): AstSelect | null { + if (seen.has(node) || count >= MAX_QUERY_BINDING_BLOCKS) { + return null; + } + seen.add(node); + count += 1; + const result: AstSelect = { + children: [], + node, + skeleton: null, + }; + const withItems = arrayProperty(node, "with"); + if (withItems === null) { + return null; + } + for (const item of withItems) { + if (item === null || typeof item !== "object") { + return null; + } + const statement = astSelectNode(objectProperty(item, "stmt")); + if (statement === null) { + return null; + } + const child = visit(statement); + if (child === null) { + return null; + } + result.children.push(child); + } + textual.push(result); + const fromItems = arrayProperty(node, "from"); + if (fromItems === null) { + return null; + } + for (const item of fromItems) { + if (item === null || typeof item !== "object") { + return null; + } + const statement = derivedAst(item); + if (statement !== null) { + const child = visit(statement); + if (child === null) { + return null; + } + result.children.push(child); + } + } + return result; + } + + const tree = visit(root); + return tree === null ? null : { root: tree, textual }; +} + +function assignSkeletons( + tree: { readonly root: AstSelect; readonly textual: readonly AstSelect[] }, + skeletons: readonly SelectSkeleton[], +): boolean { + if (tree.textual.length > skeletons.length) { + return false; + } + for (let index = 0; index < tree.textual.length; index += 1) { + const ast = phaseValue(tree.textual, index); + const skeleton = phaseValue(skeletons, index); + ast.skeleton = skeleton; + } + return true; +} + +function emitBlocks( + root: AstSelect, + statementLength: number, +): { + readonly blocks: readonly BlockBuild[]; + readonly ids: ReadonlyMap; +} | null { + const blocks: BlockBuild[] = []; + const ids = new Map(); + function emit(ast: AstSelect, parent: number | null): void { + const skeleton = assignedSkeleton(ast); + const id = blocks.length; + ids.set(ast.node, id); + blocks.push({ + ast, + id, + parent, + range: Object.freeze( + parent === null + ? { from: 0, to: statementLength } + : { from: skeleton.from, to: skeleton.to }, + ), + skeleton, + }); + for (const child of ast.children) { + emit(child, id); + } + } + try { + emit(root, null); + return { blocks, ids }; + } catch { + return null; + } +} + +function tokenIdentifier(token: Token): SqlIdentifierComponent | null { + if (token.kind === "word") { + return Object.freeze({ quoted: false, value: token.text }); + } + if (token.kind !== "quoted-identifier" || token.text.length < 2) { + return null; + } + const quote = token.text[0]; + const value = token.text + .slice(1, -1) + .split(`${quote}${quote}`) + .join(quote); + return value.length === 0 + ? null + : Object.freeze({ quoted: true, value }); +} + +function matchingClose( + tokens: readonly Token[], + openIndex: number, + openDepth: number, +): number | null { + for (let index = openIndex + 1; index < tokens.length; index += 1) { + const token = tokens[index]; + if (token?.text === ")" && token.depth === openDepth) { + return index; + } + } + return null; +} + +function relationSites( + tokens: readonly Token[], + skeleton: SelectSkeleton, +): readonly RelationSite[] { + const sites: RelationSite[] = []; + let inFrom = false; + for ( + let index = skeleton.selectToken + 1; + index < tokens.length; + index += 1 + ) { + const token = tokens[index]; + if (!token || token.from >= skeleton.to) { + break; + } + if (token.depth !== skeleton.depth) { + continue; + } + const lower = token.kind === "word" + ? token.text.toLowerCase() + : ""; + if ( + lower === "where" || + lower === "group" || + lower === "having" || + lower === "qualify" || + lower === "order" || + lower === "limit" || + lower === "window" || + lower === "union" || + lower === "intersect" || + lower === "except" + ) { + inFrom = false; + } + const startsRelation = + lower === "from" || + lower === "join" || + (inFrom && token.text === ","); + if (!startsRelation) { + continue; + } + inFrom = true; + const startIndex = index + 1; + const start = tokens[startIndex]; + if (!start) { + continue; + } + let endIndex = startIndex; + let derived = false; + if (start.text === "(") { + const close = matchingClose(tokens, startIndex, start.depth); + if (close === null) { + continue; + } + endIndex = close; + derived = true; + } else { + while (endIndex + 2 < tokens.length) { + const dot = tokens[endIndex + 1]; + const component = tokens[endIndex + 2]; + if ( + dot?.text !== "." || + component === undefined || + tokenIdentifier(component) === null + ) { + break; + } + endIndex += 2; + } + } + const lexicalIdentifiers: SqlIdentifierComponent[] = []; + if (!derived) { + for ( + let componentIndex = startIndex; + componentIndex <= endIndex; + componentIndex += 1 + ) { + const componentToken = phaseValue(tokens, componentIndex); + const identifier = tokenIdentifier(componentToken); + if (identifier !== null) { + lexicalIdentifiers.push(identifier); + } + } + } + let aliasToken: Token | null = null; + let cursor = endIndex + 1; + const explicitAlias = word(tokens[cursor], "as"); + if (explicitAlias) { + cursor += 1; + } + const possibleAlias = tokens[cursor]; + if ( + possibleAlias !== undefined && + possibleAlias.depth === skeleton.depth && + tokenIdentifier(possibleAlias) !== null && + ![ + "cross", "full", "inner", "join", "left", "natural", + "on", "right", "using", "where", "group", "having", + "qualify", "order", "limit", "window", + ].includes(possibleAlias.text.toLowerCase()) + ) { + aliasToken = possibleAlias; + endIndex = cursor; + } + const end = phaseValue(tokens, endIndex); + sites.push(Object.freeze({ + aliasToken, + derived, + explicitAlias, + from: start.from, + lexicalIdentifiers: Object.freeze(lexicalIdentifiers), + to: end.to, + })); + } + return sites; +} + +function pathFromAst( + relation: object, + grammar: NodeSqlParserQueryBindingGrammar, + site: RelationSite, +): SqlIdentifierPath | null { + const table = stringProperty(relation, "table"); + if (table === null) { + return null; + } + const values: string[] = []; + const database = + stringProperty(relation, "db") ?? + stringProperty(relation, "schema"); + if (database !== null) { + values.push(database); + } + if (grammar === "bigquery") { + values.push(...table.split(".")); + } else { + values.push(table); + } + if ( + values.length === 0 || + values.length > 8 || + values.some((value) => value.length === 0 || value.length > 256) + ) { + return null; + } + const lexicalIdentifiers = site.lexicalIdentifiers; + const allQuoted = + grammar === "bigquery" && + lexicalIdentifiers.length === 1 && + lexicalIdentifiers[0]?.quoted === true; + return Object.freeze(values.map((value, index) => + Object.freeze({ + quoted: allQuoted || lexicalIdentifiers[index]?.quoted === true, + value, + }) + )); +} + +function cteDeclarations(root: object, tokens: readonly Token[]) { + const declarations = new Map(); + const withItems = arrayProperty(root, "with"); + if (withItems === null) { + return null; + } + for (const item of withItems) { + if (item === null || typeof item !== "object") { + return null; + } + const nameObject = objectProperty(item, "name"); + const name = nameObject === null + ? stringProperty(item, "name") + : stringProperty(nameObject, "value"); + if (name === null) { + return null; + } + const token = tokens.find((candidate) => + candidate.kind === "word" && + candidate.text.toLowerCase() === name.toLowerCase() + ); + if (token === undefined) { + return null; + } + declarations.set( + name.toLowerCase(), + Object.freeze({ from: token.from, to: token.to }), + ); + } + return declarations; +} + +function addIssue( + issues: SqlQueryBindingIssue[], + code: SqlQueryBindingIssue["code"], + range: SqlTextRange, +): void { + if ( + issues.length < 256 && + !issues.some((issue) => + issue.code === code && + issue.range.from === range.from && + issue.range.to === range.to + ) + ) { + issues.push(Object.freeze({ code, range })); + } +} + +function clauseRegions( + tokens: readonly Token[], + block: BlockBuild, + fullScope: number, +): SqlVisibilityRegion[] { + const markers: { + readonly from: number; + readonly kind: SqlVisibilityRegion["kind"]; + readonly to: number; + }[] = []; + const start = block.skeleton.selectToken; + for (let index = start; index < tokens.length; index += 1) { + const token = tokens[index]; + if (!token || token.from >= block.skeleton.to) { + break; + } + if (token.depth !== block.skeleton.depth || token.kind !== "word") { + continue; + } + const value = token.text.toLowerCase(); + const kind = + value === "select" ? "select-list" + : value === "where" ? "where" + : value === "group" && word(tokens[index + 1], "by") ? "group-by" + : value === "having" ? "having" + : value === "qualify" ? "qualify" + : value === "order" && word(tokens[index + 1], "by") ? "order-by" + : value === "limit" ? "limit" + : null; + if (kind !== null) { + markers.push({ from: token.from, kind, to: token.to }); + } + } + const fromToken = tokens.find((token) => + token.depth === block.skeleton.depth && + token.from > block.skeleton.from && + token.from < block.skeleton.to && + word(token, "from") + ); + const result: SqlVisibilityRegion[] = []; + for (let index = 0; index < markers.length; index += 1) { + const marker = phaseValue(markers, index); + const next = markers[index + 1]; + const to = + marker.kind === "select-list" && fromToken !== undefined + ? fromToken.from + : next?.from ?? block.skeleton.to; + if (marker.to < to) { + result.push(Object.freeze({ + block: block.id, + kind: marker.kind, + range: Object.freeze({ from: marker.to, to }), + scope: fullScope, + })); + } + } + return result; +} + +function joinRegions( + tokens: readonly Token[], + block: BlockBuild, + scopesAfterRelations: readonly number[], +): SqlVisibilityRegion[] { + const result: SqlVisibilityRegion[] = []; + let relationIndex = -1; + for ( + let index = block.skeleton.selectToken; + index < tokens.length; + index += 1 + ) { + const token = tokens[index]; + if (!token || token.from >= block.skeleton.to) { + break; + } + if (token.depth !== block.skeleton.depth) { + continue; + } + if (word(token, "from") || word(token, "join") || token.text === ",") { + relationIndex += 1; + } + if (!word(token, "on")) { + continue; + } + let to = block.skeleton.to; + for (let cursor = index + 1; cursor < tokens.length; cursor += 1) { + const next = tokens[cursor]; + if (!next || next.from >= block.skeleton.to) { + break; + } + if ( + next.depth === block.skeleton.depth && + (word(next, "join") || + word(next, "where") || + word(next, "group") || + word(next, "having") || + word(next, "qualify") || + word(next, "order") || + word(next, "limit")) + ) { + to = next.from; + break; + } + } + const scope = scopesAfterRelations[relationIndex]; + if (scope !== undefined && token.to < to) { + result.push(Object.freeze({ + block: block.id, + kind: "join-condition", + range: Object.freeze({ from: token.to, to }), + scope, + })); + } + } + return result; +} + +export function normalizeNodeSqlParserQueryBindings( + root: unknown, + statementText: unknown, + authority: unknown, + options: NodeSqlParserQueryBindingOptions, +): NodeSqlParserQueryBindingResult { + if ( + typeof statementText !== "string" || + statementText.length === 0 || + statementText.length > 16 * 1024 || + authority === null || + typeof authority !== "object" + ) { + return Object.freeze({ reason: "resource-limit", status: "unavailable" }); + } + const astRoot = astSelectNode(root); + if (astRoot === null) { + return Object.freeze({ reason: "not-query", status: "unavailable" }); + } + const tokens = tokenize(statementText, options.grammar); + if (tokens === null) { + return Object.freeze({ reason: "resource-limit", status: "unavailable" }); + } + const skeletons = selectSkeletons(tokens, statementText.length); + const tree = buildAstTree(astRoot); + if (tree === null) { + return Object.freeze({ reason: "malformed-ast", status: "unavailable" }); + } + if (!assignSkeletons(tree, skeletons)) { + return Object.freeze({ reason: "unsupported-shape", status: "unavailable" }); + } + const emitted = emitBlocks(tree.root, statementText.length); + if (emitted === null) { + return Object.freeze({ reason: "unsupported-shape", status: "unavailable" }); + } + const declarations = cteDeclarations(astRoot, tokens); + if (declarations === null) { + return Object.freeze({ reason: "malformed-ast", status: "unavailable" }); + } + + const coverage: MutableCoverage = { + queryBlocks: + skeletons.length === tree.textual.length ? "complete" : "partial", + relationBindings: "complete", + visibility: "complete", + }; + const issues: SqlQueryBindingIssue[] = []; + const statementRange = Object.freeze({ from: 0, to: statementText.length }); + if (options.compatibility) { + coverage.queryBlocks = "partial"; + coverage.relationBindings = "partial"; + coverage.visibility = "partial"; + addIssue(issues, "parser-compatibility", statementRange); + } + if (skeletons.length !== tree.textual.length) { + addIssue(issues, "unsupported-clause", statementRange); + } + + const bindings: SqlRelationBinding[] = []; + const scopes: SqlRelationScope[] = [ + Object.freeze({ addedBinding: null, parentScope: null }), + ]; + const regions: SqlVisibilityRegion[] = []; + const publicBlocks = emitted.blocks.map((block) => + Object.freeze({ + baseScope: 0, + kind: "select" as const, + parentBlock: block.parent, + range: block.range, + }), + ); + + for (const block of emitted.blocks) { + const fromItems = arrayProperty(block.ast.node, "from"); + if (fromItems === null) { + return Object.freeze({ reason: "malformed-ast", status: "unavailable" }); + } + const sites = relationSites(tokens, block.skeleton); + if (sites.length !== fromItems.length) { + coverage.relationBindings = "partial"; + coverage.visibility = "partial"; + addIssue(issues, "unsupported-relation-source", block.range); + } + let currentScope = 0; + const after: number[] = []; + const count = Math.min(sites.length, fromItems.length); + for (let index = 0; index < count; index += 1) { + if (bindings.length >= MAX_QUERY_BINDINGS) { + return Object.freeze({ reason: "resource-limit", status: "unavailable" }); + } + const rawRelation = fromItems[index]; + const site = sites[index]; + if ( + rawRelation === null || + typeof rawRelation !== "object" || + site === undefined + ) { + return Object.freeze({ reason: "malformed-ast", status: "unavailable" }); + } + const aliasText = stringProperty(rawRelation, "as"); + const aliasToken = site.aliasToken; + const aliasName = + aliasToken === null ? null : tokenIdentifier(aliasToken); + const alias = + aliasText !== null && aliasName !== null && aliasToken !== null + ? Object.freeze({ + explicit: site.explicitAlias, + name: Object.freeze({ + quoted: aliasName.quoted, + value: aliasText, + }), + range: Object.freeze({ + from: aliasToken.from, + to: aliasToken.to, + }), + }) + : null; + if ((aliasText === null) !== (site.aliasToken === null)) { + coverage.relationBindings = "partial"; + addIssue(issues, "unsupported-relation-source", { + from: site.from, + to: site.to, + }); + } + const nested = derivedAst(rawRelation); + let source: SqlRelationBinding["source"]; + if (nested !== null) { + const nestedId = emitted.ids.get(nested); + if (nestedId === undefined || !site.derived) { + coverage.relationBindings = "partial"; + source = Object.freeze({ + kind: "unknown", + reason: "unsupported-relation-source", + }); + } else { + source = Object.freeze({ block: nestedId, kind: "derived" }); + } + } else { + const path = pathFromAst( + rawRelation, + options.grammar, + site, + ); + if (path === null || site.derived) { + coverage.relationBindings = "partial"; + source = Object.freeze({ + kind: "unknown", + reason: "unsupported-relation-source", + }); + } else { + const last = phaseValue(path, path.length - 1); + const declaration = declarations.get(last.value.toLowerCase()); + source = + declaration === undefined + ? Object.freeze({ kind: "named", path }) + : Object.freeze({ + declarationRange: declaration, + kind: "cte", + name: last, + }); + } + } + const bindingIndex = bindings.length; + bindings.push(Object.freeze({ + alias, + owner: block.id, + range: Object.freeze({ from: site.from, to: site.to }), + source, + })); + scopes.push(Object.freeze({ + addedBinding: bindingIndex, + parentScope: currentScope, + })); + if (!site.derived) { + regions.push(Object.freeze({ + block: block.id, + kind: "from-source", + range: Object.freeze({ from: site.from, to: site.to }), + scope: currentScope, + })); + } + currentScope = scopes.length - 1; + after.push(currentScope); + } + regions.push(...clauseRegions(tokens, block, currentScope)); + regions.push(...joinRegions(tokens, block, after)); + } + + regions.sort((left, right) => + left.range.from - right.range.from || + left.range.to - right.range.to + ); + const nonOverlapping: SqlVisibilityRegion[] = []; + let end = 0; + for (const region of regions) { + if (region.range.from < end) { + coverage.visibility = "partial"; + addIssue(issues, "unsupported-clause", region.range); + continue; + } + nonOverlapping.push(region); + end = region.range.to; + } + + try { + return Object.freeze({ + model: createSqlQueryBindingModel( + statementText, + authority, + { + bindings, + blocks: publicBlocks, + coverage, + issues, + regions: nonOverlapping, + scopes, + statementRange, + }, + ), + status: "ready", + }); + } catch { + return Object.freeze({ reason: "unsupported-shape", status: "unavailable" }); + } +} diff --git a/src/vnext/node-sql-parser-wire.ts b/src/vnext/node-sql-parser-wire.ts index 93cc22d..40613c0 100644 --- a/src/vnext/node-sql-parser-wire.ts +++ b/src/vnext/node-sql-parser-wire.ts @@ -2,9 +2,15 @@ import { MAX_NODE_SQL_PARSER_STATEMENT_LENGTH, type NodeSqlParserBackendOutcome, } from "./node-sql-parser-backend.js"; +import { + createSqlQueryBindingModel, + isSqlQueryBindingModel, + type SqlQueryBindingModel, + type SqlQueryBindingModelData, +} from "./query-binding-model.js"; import type { SqlStatementKind } from "./syntax.js"; -export const NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION = 1 as const; +export const NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION = 2 as const; // The parse request is the largest closed protocol shape. const MAX_NODE_SQL_PARSER_WIRE_RECORD_KEYS = 5; @@ -17,7 +23,7 @@ export type NodeSqlParserWireFailureCode = | "module-load"; export interface NodeSqlParserWireRequest { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "parse"; readonly requestId: number; readonly grammar: NodeSqlParserWireGrammar; @@ -26,34 +32,35 @@ export interface NodeSqlParserWireRequest { export type NodeSqlParserWireMessage = | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "ready"; } | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "parsed"; + readonly queryBindings: SqlQueryBindingModelData | null; readonly requestId: number; readonly statementKind: SqlStatementKind; } | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "syntax-rejected"; readonly requestId: number; } | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "unsupported"; readonly requestId: number; readonly reason: "multiple-statements" | "resource-limit"; } | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "failed"; readonly requestId: number; readonly code: NodeSqlParserWireFailureCode; } | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "protocol-error"; readonly code: "invalid-request"; }; @@ -174,6 +181,62 @@ function isRequestText(value: unknown): value is string { ); } +const wireBindingAuthority = Object.freeze({}); + +type DecodedWireBindings = + | { readonly valid: false } + | { + readonly model: SqlQueryBindingModel | null; + readonly valid: true; + }; + +function decodeWireBindings(value: unknown): DecodedWireBindings { + if (value === null) { + return { model: null, valid: true }; + } + if (value === null || typeof value !== "object") { + return { valid: false }; + } + try { + const statementRange = Object.getOwnPropertyDescriptor( + value, + "statementRange", + ); + if ( + statementRange === undefined || + !("value" in statementRange) || + statementRange.value === null || + typeof statementRange.value !== "object" + ) { + return { valid: false }; + } + const to = Object.getOwnPropertyDescriptor( + statementRange.value, + "to", + ); + if ( + to === undefined || + !("value" in to) || + typeof to.value !== "number" || + !Number.isSafeInteger(to.value) || + to.value <= 0 || + to.value > MAX_NODE_SQL_PARSER_STATEMENT_LENGTH + ) { + return { valid: false }; + } + return { + model: createSqlQueryBindingModel( + " ".repeat(to.value), + wireBindingAuthority, + value, + ), + valid: true, + }; + } catch { + return { valid: false }; + } +} + function requireRequestId(value: number): void { if (!isRequestId(value)) { throw new TypeError( @@ -299,6 +362,7 @@ export function encodeNodeSqlParserWireProtocolError(): Extract< export function encodeNodeSqlParserWireBackendOutcome( requestId: number, outcome: NodeSqlParserBackendOutcome, + queryBindings: SqlQueryBindingModel | null = null, ): Exclude< NodeSqlParserWireMessage, { readonly kind: "protocol-error" | "ready" } @@ -307,9 +371,18 @@ export function encodeNodeSqlParserWireBackendOutcome( switch (outcome.kind) { case "parsed": requireStatementKind(outcome.statementKind); + if ( + queryBindings !== null && + !isSqlQueryBindingModel(queryBindings) + ) { + throw new TypeError( + "node-sql-parser wire query bindings must be authenticated", + ); + } return Object.freeze({ kind: "parsed", protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, + queryBindings, requestId, statementKind: outcome.statementKind, }); @@ -362,21 +435,27 @@ export function decodeNodeSqlParserWireMessage( : null; case "parsed": { const statementKind = record.values.get("statementKind"); + const queryBindings = decodeWireBindings( + record.values.get("queryBindings"), + ); if ( !hasExactKeys(record, [ "protocolVersion", "kind", + "queryBindings", "requestId", "statementKind", ]) || !isRequestId(requestId) || - !isStatementKind(statementKind) + !isStatementKind(statementKind) || + !queryBindings.valid ) { return null; } return Object.freeze({ kind: "parsed", protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, + queryBindings: queryBindings.model, requestId, statementKind, }); diff --git a/src/vnext/query-binding-model.ts b/src/vnext/query-binding-model.ts index 8aeb4a4..924e9da 100644 --- a/src/vnext/query-binding-model.ts +++ b/src/vnext/query-binding-model.ts @@ -113,7 +113,8 @@ export interface SqlRelationBinding { readonly source: SqlRelationBindingSource; } -export interface SqlQueryBindingModel { +/** Validated plain-data shape used before exact source/authority authentication. */ +export interface SqlQueryBindingModelData { readonly blocks: readonly SqlQueryBlock[]; readonly bindings: readonly SqlRelationBinding[]; readonly coverage: SqlQueryBindingCoverageSet; @@ -123,6 +124,9 @@ export interface SqlQueryBindingModel { readonly statementRange: SqlTextRange; } +/** A model authenticated for one exact statement and parser authority. */ +export interface SqlQueryBindingModel extends SqlQueryBindingModelData {} + export type SqlQueryBindingModelErrorCode = | "invalid-authority" | "invalid-model" From f9eb7db44198974563265b6eaaef38e89f572235 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:51:28 +0800 Subject: [PATCH 08/25] feat(vnext): complete columns from query bindings --- src/vnext/__tests__/column-completion.test.ts | 420 ++++++++++++++ src/vnext/__tests__/column-query-site.test.ts | 325 +++++++++++ .../__tests__/relation-completion.test.ts | 15 +- src/vnext/__tests__/session.test.ts | 189 +++++++ .../sql-editor-completion-gate.test.ts | 67 +++ src/vnext/column-completion.ts | 320 +++++++++++ src/vnext/column-query-site.ts | 528 ++++++++++++++++++ src/vnext/index.ts | 14 + src/vnext/local-relation-site.ts | 32 ++ src/vnext/relation-completion-types.ts | 54 +- src/vnext/relation-completion.ts | 2 +- src/vnext/session.ts | 198 ++++++- src/vnext/types.ts | 4 + 13 files changed, 2160 insertions(+), 8 deletions(-) create mode 100644 src/vnext/__tests__/column-completion.test.ts create mode 100644 src/vnext/__tests__/column-query-site.test.ts create mode 100644 src/vnext/column-completion.ts create mode 100644 src/vnext/column-query-site.ts diff --git a/src/vnext/__tests__/column-completion.test.ts b/src/vnext/__tests__/column-completion.test.ts new file mode 100644 index 0000000..4cf0c9d --- /dev/null +++ b/src/vnext/__tests__/column-completion.test.ts @@ -0,0 +1,420 @@ +import { describe, expect, it } from "vitest"; +import { + composeSqlColumnCompletion, + prepareSqlColumnCatalogRelations, +} from "../column-completion.js"; +import type { + SqlColumnCatalogBatchOutcome, +} from "../column-catalog-batch-coordinator.js"; +import type { + SqlColumnCatalogResolvedColumn, +} from "../column-catalog-types.js"; +import type { + SqlColumnQuerySiteResult, +} from "../column-query-site.js"; +import { + POSTGRESQL_SQL_RELATION_DIALECT, +} from "../relation-dialect.js"; + +type ReadySite = Extract< + SqlColumnQuerySiteResult, + { status: "ready" } +>; + +const epoch = Object.freeze({ generation: 1, token: "epoch-1" }); + +function site( + qualifier: ReadySite["qualifier"] = [], + prefix = "", +): ReadySite { + return Object.freeze({ + coverage: "complete", + issues: Object.freeze([]), + prefix: Object.freeze({ quoted: false, value: prefix }), + qualifier: Object.freeze(qualifier), + relations: Object.freeze([ + Object.freeze({ + alias: Object.freeze({ quoted: false, value: "u" }), + path: Object.freeze([ + Object.freeze({ quoted: false, value: "app" }), + Object.freeze({ quoted: false, value: "users" }), + ]), + range: Object.freeze({ from: 20, to: 29 }), + }), + Object.freeze({ + alias: Object.freeze({ quoted: false, value: "o" }), + path: Object.freeze([ + Object.freeze({ quoted: false, value: "orders" }), + ]), + range: Object.freeze({ from: 35, to: 41 }), + }), + ]), + replacementRange: Object.freeze({ from: 9, to: 9 + prefix.length }), + status: "ready", + }); +} + +function column( + name: string, + relationEntityId: string, + ordinal: number, +): SqlColumnCatalogResolvedColumn { + return Object.freeze({ + columnEntityId: `${relationEntityId}:${name}`, + dataType: "VARCHAR", + identifier: Object.freeze({ quoted: false, value: name }), + insertText: name, + ordinal, + provenance: Object.freeze({ + columnEntityId: `${relationEntityId}:${name}`, + epoch, + providerId: "columns", + relationEntityId, + scope: "engine:1", + }), + }); +} + +function usable( + relations: Extract< + SqlColumnCatalogBatchOutcome, + { status: "usable" } + >["relations"], +): Extract { + return Object.freeze({ + epoch, + providerId: "columns", + relations: Object.freeze(relations), + scope: "engine:1", + status: "usable", + }); +} + +describe("column completion", () => { + it("batches only the relation selected by an alias qualifier", () => { + const current = site([{ quoted: false, value: "u" }], "na"); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + + expect(prepared.references).toEqual([{ + path: [ + { quoted: false, value: "app" }, + { quoted: false, value: "users" }, + ], + requestKey: "binding:0", + }]); + }); + + it("matches an unaliased multipart path suffix", () => { + const current = Object.freeze({ + ...site([{ quoted: false, value: "app" }, { + quoted: false, + value: "users", + }]), + relations: Object.freeze([ + Object.freeze({ + alias: null, + path: Object.freeze([ + Object.freeze({ quoted: false, value: "app" }), + Object.freeze({ quoted: false, value: "users" }), + ]), + range: Object.freeze({ from: 20, to: 29 }), + }), + ]), + }); + expect( + prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ).references, + ).toHaveLength(1); + }); + + it("composes deterministic exact edits and provenance", () => { + const current = site([], "na"); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const result = composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([ + { + columns: [ + column("nickname", "users", 2), + column("name", "users", 1), + ], + coverage: "complete", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }, + { + columns: [column("name", "orders", 1)], + coverage: "complete", + relationEntityId: "orders", + requestKey: "binding:1", + status: "ready", + }, + ]), + prepared, + providerId: "columns", + site: current, + }); + + expect(result?.value).toMatchObject({ + isIncomplete: false, + items: [ + { + detail: "VARCHAR — o", + edit: { from: 9, insert: "name", to: 11 }, + kind: "column", + label: "name", + provenance: { + columnEntityId: "orders:name", + kind: "column-catalog", + relationEntityId: "orders", + }, + }, + { + detail: "VARCHAR — u", + label: "name", + }, + ], + }); + }); + + it("reports partial, loading, and failed relation evidence", () => { + const current = Object.freeze({ + ...site(), + coverage: "partial" as const, + issues: Object.freeze(["derived-relation" as const]), + }); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const result = composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([ + { + columns: [column("id", "users", 0)], + coverage: "partial", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }, + { requestKey: "binding:1", status: "loading" }, + ]), + prepared, + providerId: "columns", + site: current, + }); + + expect(result?.value).toMatchObject({ + isIncomplete: true, + issues: [ + { reason: "query-binding-partial" }, + { reason: "column-catalog-partial" }, + { reason: "column-catalog-loading" }, + ], + }); + expect(result?.sources[0]).toMatchObject({ + feature: "column-catalog", + outcome: "loading", + }); + }); + + it("maps provider unavailability and suppresses cancellations", () => { + const current = site(); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: { + reason: "malformed-response", + status: "unavailable", + }, + prepared, + providerId: "columns", + site: current, + }), + ).toMatchObject({ + value: { + issues: [{ reason: "column-catalog-malformed" }], + }, + }); + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: { status: "cancelled" }, + prepared, + providerId: "columns", + site: current, + }), + ).toBeNull(); + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: { status: "superseded" }, + prepared, + providerId: "columns", + site: current, + }), + ).toBeNull(); + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: { + reason: "provider-failed", + status: "unavailable", + }, + prepared, + providerId: "columns", + site: current, + }), + ).toMatchObject({ + value: { + issues: [{ reason: "column-catalog-failed" }], + }, + }); + }); + + it("fails qualifier matching closed and labels unaliased relations", () => { + const current = Object.freeze({ + ...site([ + { quoted: false, value: "too" }, + { quoted: false, value: "deep" }, + { quoted: false, value: "path" }, + ]), + relations: Object.freeze([ + Object.freeze({ + alias: null, + path: Object.freeze([ + Object.freeze({ quoted: false, value: "app" }), + Object.freeze({ quoted: false, value: "users" }), + ]), + range: Object.freeze({ from: 20, to: 29 }), + }), + ]), + }); + expect( + prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ).references, + ).toEqual([]); + + const unqualified = Object.freeze({ + ...current, + qualifier: Object.freeze([]), + }); + const prepared = prepareSqlColumnCatalogRelations( + unqualified, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const noMetadata = Object.freeze({ + columnEntityId: "users:x", + identifier: Object.freeze({ quoted: false, value: "x" }), + insertText: "x", + ordinal: 0, + provenance: Object.freeze({ + columnEntityId: "users:x", + epoch, + providerId: "columns", + relationEntityId: "users", + scope: "engine:1", + }), + }); + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([{ + columns: [noMetadata], + coverage: "complete", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }]), + prepared, + providerId: "columns", + site: unqualified, + })?.value.items[0], + ).toMatchObject({ + detail: "app.users", + label: "x", + }); + }); + + it("contains malformed mappings, duplicates, prefix misses, and failures", () => { + const current = site([], "i"); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const repeated = column("id", "users", 0); + const result = composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([ + { + columns: [repeated, repeated, column("name", "users", 1)], + coverage: "complete", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }, + { + columns: [column("id", "missing", 0)], + coverage: "complete", + relationEntityId: "missing", + requestKey: "unknown", + status: "ready", + }, + { + code: "unavailable", + requestKey: "binding:1", + retry: "next-request", + status: "failed", + }, + ]), + prepared, + providerId: "columns", + site: current, + }); + + expect(result).toMatchObject({ + sources: [{ + coverage: "partial", + outcome: "ready", + }], + value: { + isIncomplete: true, + issues: [ + { reason: "column-catalog-malformed" }, + { reason: "column-catalog-failed" }, + ], + items: [{ label: "id" }], + }, + }); + + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([{ + code: "unavailable", + requestKey: "binding:0", + retry: "next-request", + status: "failed", + }]), + prepared, + providerId: "columns", + site: current, + })?.sources[0], + ).toMatchObject({ outcome: "failed" }); + }); +}); diff --git a/src/vnext/__tests__/column-query-site.test.ts b/src/vnext/__tests__/column-query-site.test.ts new file mode 100644 index 0000000..6fd9eb1 --- /dev/null +++ b/src/vnext/__tests__/column-query-site.test.ts @@ -0,0 +1,325 @@ +import { describe, expect, it } from "vitest"; +import { MAX_BOUNDED_SQL_LEXEMES } from "../bounded-sql-lexer.js"; +import { + MAX_COLUMN_QUERY_RELATIONS, + recognizeSqlColumnQuerySite, + type SqlColumnQuerySiteResult, +} from "../column-query-site.js"; +import { + BIGQUERY_SQL_RELATION_DIALECT, + POSTGRESQL_SQL_RELATION_DIALECT, + type SqlRelationDialectRuntime, +} from "../relation-dialect.js"; +import { + createIdentitySqlSource, + createMaskedSqlSource, +} from "../source.js"; +import { + buildSqlStatementIndex, + findSqlStatementSlot, +} from "../statement-index.js"; + +function analyze( + marked: string, + options: { + readonly dialect?: SqlRelationDialectRuntime; + readonly regions?: readonly { + readonly from: number; + readonly language: string; + readonly to: number; + }[]; + } = {}, +): SqlColumnQuerySiteResult { + const position = marked.indexOf("|"); + if (position < 0 || marked.indexOf("|", position + 1) >= 0) { + throw new Error("Fixture requires exactly one cursor marker"); + } + const text = marked.slice(0, position) + marked.slice(position + 1); + const dialect = options.dialect ?? POSTGRESQL_SQL_RELATION_DIALECT; + const source = options.regions + ? createMaskedSqlSource(text, options.regions) + : createIdentitySqlSource(text); + const index = buildSqlStatementIndex( + source.analysisText, + dialect.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot(index, position, "left"); + return recognizeSqlColumnQuerySite( + source, + slot, + position, + dialect, + ); +} + +function ready( + result: SqlColumnQuerySiteResult, +): Extract { + if (result.status !== "ready") { + throw new Error(`Expected ready, received ${JSON.stringify(result)}`); + } + expect(result.status).toBe("ready"); + return result; +} + +describe("recognizeSqlColumnQuerySite", () => { + it("finds a qualified SELECT-list site and its aliased relation", () => { + const result = ready( + analyze("SELECT u.na| FROM app.users AS u"), + ); + + expect(result).toMatchObject({ + coverage: "complete", + prefix: { quoted: false, value: "na" }, + qualifier: [{ quoted: false, value: "u" }], + replacementRange: { from: 9, to: 11 }, + }); + expect(result.relations).toEqual([{ + alias: { quoted: false, value: "u" }, + path: [ + { quoted: false, value: "app" }, + { quoted: false, value: "users" }, + ], + range: { from: 17, to: 26 }, + }]); + }); + + it("collects comma and join relations for expression clauses", () => { + const result = ready( + analyze( + "SELECT * FROM users u, ignored i JOIN orders AS o ON o.id = u.id WHERE cr|", + ), + ); + + expect(result.prefix.value).toBe("cr"); + expect(result.qualifier).toEqual([]); + expect(result.relations.map((relation) => relation.alias?.value)) + .toEqual(["u", "i", "o"]); + }); + + it("uses the innermost query block", () => { + const result = ready( + analyze( + "SELECT * FROM outer_table o WHERE EXISTS (SELECT i.na| FROM inner_table i)", + ), + ); + + expect(result.qualifier[0]?.value).toBe("i"); + expect(result.relations).toHaveLength(1); + expect(result.relations[0]?.path.at(-1)?.value).toBe( + "inner_table", + ); + }); + + it("supports BigQuery quoted multipart bindings", () => { + const result = ready( + analyze("SELECT t.| FROM `project.dataset.table` AS t", { + dialect: BIGQUERY_SQL_RELATION_DIALECT, + }), + ); + + expect(result.relations[0]).toMatchObject({ + alias: { quoted: false, value: "t" }, + path: [ + { quoted: true, value: "project" }, + { quoted: true, value: "dataset" }, + { quoted: true, value: "table" }, + ], + }); + }); + + it.each([ + ["SELECT * FROM us|", "not-column-position"], + ["INSERT INTO users VALUES (|)", "not-select-query"], + ["SELECT 'abc|' FROM users", "cursor-in-string"], + ["SELECT /* abc| */ 1 FROM users", "cursor-in-comment"], + ] as const)("fails closed for %s", (marked, reason) => { + expect(analyze(marked)).toMatchObject({ reason, status: "inactive" }); + }); + + it("fails closed inside an embedded region", () => { + const marked = "SELECT {value|} FROM users"; + const position = marked.indexOf("|"); + expect( + analyze(marked, { + regions: [{ + from: marked.indexOf("{"), + language: "python", + to: marked.indexOf("}") + 1, + }], + }), + ).toMatchObject({ + reason: "cursor-in-embedded-region", + status: "inactive", + }); + expect(position).toBeGreaterThan(0); + }); + + it("marks derived and table-function evidence partial", () => { + expect( + ready( + analyze( + "SELECT x.| FROM (SELECT 1) x JOIN UNNEST(items) AS item ON true", + ), + ), + ).toMatchObject({ + coverage: "partial", + issues: [ + "derived-relation", + "nested-query", + "table-function", + ], + }); + }); + + it.each([ + "SELECT * FROM users WHERE na|", + "SELECT * FROM users GROUP BY na|", + "SELECT * FROM users HAVING na|", + "SELECT * FROM users QUALIFY na|", + "SELECT * FROM users ORDER BY na|", + "SELECT * FROM users u JOIN orders o ON o.id = u.|", + "SELECT * FROM users u JOIN orders o USING (i|)", + ])("recognizes expression clause sites in %s", (marked) => { + expect(ready(analyze(marked)).relations.length).toBeGreaterThan(0); + }); + + it.each([ + "SELECT * FROM users LIMIT |", + "SELECT * FROM users FETCH |", + "SELECT * FROM users OFFSET |", + ])("rejects row-limit clause sites in %s", (marked) => { + expect(analyze(marked)).toMatchObject({ + reason: "not-column-position", + status: "inactive", + }); + }); + + it("supports implicit quoted aliases and incomplete relations", () => { + expect( + ready(analyze('SELECT "u".| FROM app.users "u"')).relations[0], + ).toMatchObject({ + alias: { quoted: true, value: "u" }, + }); + expect(ready(analyze("SELECT | FROM"))).toMatchObject({ + coverage: "partial", + issues: ["incomplete-relation"], + relations: [], + }); + expect(ready(analyze('SELECT | FROM ""'))).toMatchObject({ + coverage: "partial", + issues: ["incomplete-relation"], + relations: [], + }); + }); + + it("fails closed for malformed typed paths and line comments", () => { + expect(analyze("SELECT u..| FROM users u")).toMatchObject({ + reason: "not-column-position", + status: "inactive", + }); + expect(analyze("SELECT 1 -- co|mment\nFROM users")).toMatchObject({ + reason: "cursor-in-comment", + status: "inactive", + }); + }); + + it("bounds lexical work before relation analysis", () => { + const expression = Array.from( + { length: MAX_BOUNDED_SQL_LEXEMES + 1 }, + () => "x", + ).join("+"); + expect(analyze(`SELECT ${expression}| FROM users`)).toMatchObject({ + reason: "resource-limit", + status: "unavailable", + }); + }); + + it("bounds relation materialization", () => { + const joins = Array.from( + { length: MAX_COLUMN_QUERY_RELATIONS + 1 }, + (_, index) => ` JOIN table_${index} t_${index} ON true`, + ).join(""); + expect(analyze(`SELECT | FROM base b${joins}`)).toMatchObject({ + reason: "resource-limit", + status: "unavailable", + }); + }); + + it("rejects foreign contract values without inspecting them", () => { + let inspected = false; + const hostile = new Proxy({}, { + get() { + inspected = true; + throw new Error("hostile"); + }, + }); + expect( + Reflect.apply(recognizeSqlColumnQuerySite, undefined, [ + hostile, + hostile, + 0, + hostile, + ]), + ).toMatchObject({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + expect(inspected).toBe(false); + }); + + it("rejects each unauthenticated argument and invalid position", () => { + const source = createIdentitySqlSource("SELECT |"); + const position = source.analysisText.indexOf("|"); + const index = buildSqlStatementIndex( + source.analysisText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot(index, position, "left"); + for (const input of [ + [{}, slot, position, POSTGRESQL_SQL_RELATION_DIALECT], + [source, {}, position, POSTGRESQL_SQL_RELATION_DIALECT], + [source, slot, position, {}], + [source, slot, Number.NaN, POSTGRESQL_SQL_RELATION_DIALECT], + [source, slot, -1, POSTGRESQL_SQL_RELATION_DIALECT], + [ + source, + slot, + source.analysisText.length + 1, + POSTGRESQL_SQL_RELATION_DIALECT, + ], + ]) { + expect( + Reflect.apply(recognizeSqlColumnQuerySite, undefined, input), + ).toMatchObject({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + } + }); + + it("preserves opaque statement failures", () => { + const source = createIdentitySqlSource("DELIMITER $$"); + const index = buildSqlStatementIndex( + source.analysisText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot( + index, + source.analysisText.length, + "left", + ); + expect(slot.boundaryQuality).toBe("opaque"); + expect( + recognizeSqlColumnQuerySite( + source, + slot, + source.analysisText.length, + POSTGRESQL_SQL_RELATION_DIALECT, + ), + ).toMatchObject({ + reason: "opaque-statement", + status: "unavailable", + }); + }); +}); diff --git a/src/vnext/__tests__/relation-completion.test.ts b/src/vnext/__tests__/relation-completion.test.ts index 8a25702..b33efba 100644 --- a/src/vnext/__tests__/relation-completion.test.ts +++ b/src/vnext/__tests__/relation-completion.test.ts @@ -232,11 +232,16 @@ describe("relation completion composition", () => { ); expect( - value.items.map((item) => [ - item.relationKind, - item.label, - item.edit.insert, - ]), + value.items.map((item) => { + if (item.kind !== "relation") { + throw new Error("Expected a relation completion"); + } + return [ + item.relationKind, + item.label, + item.edit.insert, + ]; + }), ).toEqual([ ["cte", "users", "users"], ["cte", "zed", "zed"], diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index 14c3ad0..eb64669 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -6,6 +6,7 @@ import { duckdbDialect, postgresDialect, SqlSessionError, + type SqlColumnCatalogProvider, } from "../index.js"; import { DefaultSqlLanguageService, @@ -1239,6 +1240,194 @@ describe("relation completion session integration", () => { }); }); +describe("column completion session integration", () => { + const relationCatalog: SqlRelationCatalogProvider = { + id: "relations", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + status: "ready", + }), + }; + + function serviceWithColumns( + loadColumns: SqlColumnCatalogProvider["loadColumns"], + ) { + return createSqlLanguageService({ + catalog: relationCatalog, + columns: { id: "columns", loadColumns }, + completion: { catalogResponseBudgetMs: 40 }, + dialects: [duckdb], + }); + } + + it("batches and completes a qualified alias through the session", async () => { + const requests: Parameters< + SqlColumnCatalogProvider["loadColumns"] + >[0][] = []; + const service = serviceWithColumns(async (request) => { + requests.push(request); + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: [{ + columns: [{ + columnEntityId: "users:name", + dataType: "VARCHAR", + identifier: { quoted: false, value: "name" }, + insertText: "name", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: "users", + requestKey: request.relations[0]?.requestKey, + status: "ready", + }], + }; + }); + const text = "SELECT u.na FROM app.users AS u"; + const session = service.openDocument({ + context: { + catalog: { + scope: "connection:1", + searchPath: [[{ quoted: false, value: "app" }]], + }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + + await expect( + session.complete({ + position: text.indexOf("na") + 2, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + sources: [{ + feature: "column-catalog", + outcome: "ready", + providerId: "columns", + }], + status: "ready", + value: { + isIncomplete: false, + items: [{ + dataType: "VARCHAR", + edit: { from: 9, insert: "name", to: 11 }, + kind: "column", + label: "name", + provenance: { + columnEntityId: "users:name", + kind: "column-catalog", + providerId: "columns", + relationEntityId: "users", + scope: "connection:1", + }, + }], + }, + }); + expect(requests).toHaveLength(1); + expect(requests[0]).toMatchObject({ + dialectId: "duckdb", + expectedEpoch: null, + relations: [{ + path: [ + { quoted: false, value: "app" }, + { quoted: false, value: "users" }, + ], + requestKey: "binding:0", + }], + scope: "connection:1", + }); + service.dispose(); + }); + + it("loads all visible relations in one unqualified batch", async () => { + let calls = 0; + const service = serviceWithColumns(async (request) => { + calls += 1; + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: request.relations.map((relation, index) => ({ + columns: [{ + columnEntityId: `relation-${index}:id`, + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: `relation-${index}`, + requestKey: relation.requestKey, + status: "ready", + })), + }; + }); + const text = + "SELECT i FROM users u JOIN orders o ON o.user_id = u.id"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const result = await session.complete({ + position: "SELECT i".length, + trigger: { kind: "invoked" }, + }); + + expect(calls).toBe(1); + expect(result).toMatchObject({ + status: "ready", + value: { + items: [ + { detail: "o", label: "id" }, + { detail: "u", label: "id" }, + ], + }, + }); + service.dispose(); + }); + + it("keeps slow providers inside the completion response budget", async () => { + vi.useFakeTimers(); + try { + const service = serviceWithColumns(() => new Promise(() => {})); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const completion = session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + await vi.advanceTimersByTimeAsync(40); + await expect(completion).resolves.toMatchObject({ + sources: [{ + feature: "column-catalog", + outcome: "loading", + }], + status: "ready", + value: { + isIncomplete: true, + issues: [{ reason: "column-catalog-loading" }], + items: [], + }, + }); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); +}); + describe("statement-index session cache", () => { it("binds lexical behavior through authentic dialect handles", () => { const service = new DefaultSqlLanguageService({ diff --git a/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts index 3475b1f..6e79d23 100644 --- a/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts +++ b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts @@ -151,3 +151,70 @@ test("vNext completion gate permits normal SQL in a browser editor", async () => service.dispose(); parent.remove(); }); + +test("vNext editor applies a batched column completion", async () => { + const parent = document.createElement("div"); + document.body.append(parent); + let columnCalls = 0; + const service = createSqlLanguageService({ + catalog: { + id: "browser-relations", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + status: "ready" as const, + }), + }, + columns: { + id: "browser-columns", + loadColumns: async (request) => { + columnCalls += 1; + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: [{ + columns: [{ + columnEntityId: "users:name", + dataType: "VARCHAR", + identifier: { quoted: false, value: "name" }, + insertText: "name", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: "users", + requestKey: request.relations[0]?.requestKey, + status: "ready", + }], + }; + }, + }, + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { + catalog: { scope: "browser-columns" }, + dialect: "duckdb", + }, + service, + }); + const documentText = "SELECT u.na FROM users u"; + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + selection: { anchor: "SELECT u.na".length }, + }); + + expect(startCompletion(view)).toBe(true); + await expect.poll(() => + currentCompletions(view.state).map((item) => ({ + label: item.label, + type: item.type, + })) + ).toEqual([{ label: "name", type: "property" }]); + expect(columnCalls).toBe(1); + + view.destroy(); + service.dispose(); + parent.remove(); +}); diff --git a/src/vnext/column-completion.ts b/src/vnext/column-completion.ts new file mode 100644 index 0000000..b0b249e --- /dev/null +++ b/src/vnext/column-completion.ts @@ -0,0 +1,320 @@ +import type { + SqlColumnCatalogBatchOutcome, +} from "./column-catalog-batch-coordinator.js"; +import type { + SqlColumnCatalogRelationReference, + SqlColumnCatalogRelationResult, +} from "./column-catalog-types.js"; +import type { + SqlColumnQueryRelation, + SqlColumnQuerySiteResult, +} from "./column-query-site.js"; +import type { + SqlColumnCatalogProviderReport, + SqlCompletionIssue, + SqlCompletionItem, + SqlCompletionList, +} from "./relation-completion-types.js"; +import type { SqlRelationDialectRuntime } from "./relation-dialect.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, +} from "./types.js"; + +type ReadyColumnSite = Extract< + SqlColumnQuerySiteResult, + { readonly status: "ready" } +>; + +export interface SqlPreparedColumnCatalogRelations { + readonly references: readonly SqlColumnCatalogRelationReference[]; + readonly relationsByRequestKey: ReadonlyMap< + string, + SqlColumnQueryRelation + >; +} + +export interface SqlColumnCompletionComposition { + readonly sources: readonly SqlColumnCatalogProviderReport[]; + readonly value: SqlCompletionList; +} + +function sameIdentifier( + dialect: SqlRelationDialectRuntime, + left: SqlIdentifierComponent, + right: SqlIdentifierComponent, +): boolean { + return dialect.completion.compareCteIdentifiers(left, right) === "equal"; +} + +function pathEndsWith( + dialect: SqlRelationDialectRuntime, + path: SqlIdentifierPath, + suffix: SqlIdentifierPath, +): boolean { + if (suffix.length === 0 || suffix.length > path.length) return false; + const offset = path.length - suffix.length; + return suffix.every((component, index) => { + const candidate = path[offset + index]; + return candidate !== undefined && + sameIdentifier(dialect, candidate, component); + }); +} + +function relationMatchesQualifier( + dialect: SqlRelationDialectRuntime, + relation: SqlColumnQueryRelation, + qualifier: SqlIdentifierPath, +): boolean { + if (qualifier.length === 0) return true; + if ( + qualifier.length === 1 && + relation.alias !== null && + qualifier[0] !== undefined && + sameIdentifier(dialect, relation.alias, qualifier[0]) + ) { + return true; + } + return relation.alias === null && + pathEndsWith(dialect, relation.path, qualifier); +} + +export function prepareSqlColumnCatalogRelations( + site: ReadyColumnSite, + dialect: SqlRelationDialectRuntime, +): SqlPreparedColumnCatalogRelations { + const references: SqlColumnCatalogRelationReference[] = []; + const relationsByRequestKey = new Map< + string, + SqlColumnQueryRelation + >(); + for (let index = 0; index < site.relations.length; index += 1) { + const relation = site.relations[index]; + if ( + relation === undefined || + !relationMatchesQualifier( + dialect, + relation, + site.qualifier, + ) + ) { + continue; + } + const requestKey = `binding:${index}`; + references.push(Object.freeze({ + path: relation.path, + requestKey, + })); + relationsByRequestKey.set(requestKey, relation); + } + return Object.freeze({ + references: Object.freeze(references), + relationsByRequestKey, + }); +} + +function relationLabel(relation: SqlColumnQueryRelation): string { + return relation.alias?.value ?? + relation.path.map((component) => component.value).join("."); +} + +function issue( + reason: Exclude, +): SqlCompletionIssue { + return Object.freeze({ reason }); +} + +function resultIssues( + relation: SqlColumnCatalogRelationResult, +): readonly SqlCompletionIssue[] { + if (relation.status === "loading") { + return Object.freeze([issue("column-catalog-loading")]); + } + if (relation.status === "failed") { + return Object.freeze([issue("column-catalog-failed")]); + } + return relation.coverage === "partial" + ? Object.freeze([issue("column-catalog-partial")]) + : Object.freeze([]); +} + +function list( + items: readonly SqlCompletionItem[], + issues: readonly SqlCompletionIssue[], +): SqlCompletionList { + const first = issues[0]; + if (first === undefined) { + return Object.freeze({ + isIncomplete: false, + issues: Object.freeze([] as const), + items: Object.freeze(items), + }); + } + const incompleteIssues: [ + SqlCompletionIssue, + ...SqlCompletionIssue[], + ] = [first, ...issues.slice(1)]; + return Object.freeze({ + isIncomplete: true, + issues: Object.freeze(incompleteIssues), + items: Object.freeze(items), + }); +} + +function unavailableComposition( + providerId: string, + reason: Extract< + SqlColumnCatalogBatchOutcome, + { readonly status: "unavailable" } + >["reason"], +): SqlColumnCompletionComposition { + return Object.freeze({ + sources: Object.freeze([Object.freeze({ + feature: "column-catalog", + outcome: "unavailable", + providerId, + reason, + })]), + value: list([], [issue( + reason === "malformed-response" + ? "column-catalog-malformed" + : "column-catalog-failed", + )]), + }); +} + +function compareItems( + left: SqlCompletionItem, + right: SqlCompletionItem, +): number { + return left.label.localeCompare(right.label) || + (left.detail ?? "").localeCompare(right.detail ?? "") || + left.edit.insert.localeCompare(right.edit.insert) || + left.edit.from - right.edit.from || + left.edit.to - right.edit.to; +} + +export function composeSqlColumnCompletion( + input: { + readonly dialect: SqlRelationDialectRuntime; + readonly outcome: SqlColumnCatalogBatchOutcome; + readonly prepared: SqlPreparedColumnCatalogRelations; + readonly providerId: string; + readonly site: ReadyColumnSite; + }, +): SqlColumnCompletionComposition | null { + if ( + input.outcome.status === "cancelled" || + input.outcome.status === "superseded" + ) { + return null; + } + if (input.outcome.status === "unavailable") { + return unavailableComposition( + input.providerId, + input.outcome.reason, + ); + } + const items: SqlCompletionItem[] = []; + const issues: SqlCompletionIssue[] = []; + if (input.site.coverage === "partial") { + issues.push(issue("query-binding-partial")); + } + let hasLoading = false; + let hasFailure = false; + let hasPartial = input.site.coverage === "partial"; + const seen = new Set(); + for (const result of input.outcome.relations) { + issues.push(...resultIssues(result)); + if (result.status === "loading") { + hasLoading = true; + continue; + } + if (result.status === "failed") { + hasFailure = true; + continue; + } + if (result.coverage === "partial") hasPartial = true; + const relation = input.prepared.relationsByRequestKey.get( + result.requestKey, + ); + if (!relation) { + issues.push(issue("column-catalog-malformed")); + continue; + } + for (const column of result.columns) { + if ( + input.dialect.completion.cteIdentifierMatchesPrefix( + column.identifier, + input.site.prefix, + ) !== "match" + ) { + continue; + } + const identity = [ + column.provenance.providerId, + column.provenance.scope, + column.provenance.relationEntityId, + column.provenance.columnEntityId, + column.insertText, + ].join("\u0000"); + if (seen.has(identity)) continue; + seen.add(identity); + const relationName = relationLabel(relation); + const metadata = column.detail ?? column.dataType; + items.push(Object.freeze({ + ...(column.dataType === undefined + ? {} + : { dataType: column.dataType }), + detail: metadata === undefined + ? relationName + : `${metadata} — ${relationName}`, + edit: Object.freeze({ + from: input.site.replacementRange.from, + insert: column.insertText, + to: input.site.replacementRange.to, + }), + kind: "column", + label: column.identifier.value, + provenance: Object.freeze({ + columnEntityId: column.provenance.columnEntityId, + epoch: column.provenance.epoch, + kind: "column-catalog", + providerId: column.provenance.providerId, + relationEntityId: column.provenance.relationEntityId, + scope: column.provenance.scope, + }), + relationRequestKey: result.requestKey, + })); + } + } + const deduplicatedIssues = Array.from( + new Map(issues.map((item) => [item.reason, item])).values(), + ); + const coverage = hasPartial || hasFailure ? "partial" : "complete"; + const source: SqlColumnCatalogProviderReport = hasLoading + ? { + feature: "column-catalog", + outcome: "loading", + providerId: input.outcome.providerId, + } + : hasFailure && items.length === 0 + ? { + feature: "column-catalog", + outcome: "failed", + providerId: input.outcome.providerId, + } + : { + coverage, + feature: "column-catalog", + outcome: "ready", + providerId: input.outcome.providerId, + }; + return Object.freeze({ + sources: Object.freeze([Object.freeze(source)]), + value: list( + Object.freeze(items.sort(compareItems)), + Object.freeze(deduplicatedIssues), + ), + }); +} diff --git a/src/vnext/column-query-site.ts b/src/vnext/column-query-site.ts new file mode 100644 index 0000000..9c5dc00 --- /dev/null +++ b/src/vnext/column-query-site.ts @@ -0,0 +1,528 @@ +import { + BoundedSqlLexer, + type BoundedSqlLexeme, +} from "./bounded-sql-lexer.js"; +import type { SqlRelationDialectRuntime } from "./relation-dialect.js"; +import { isSqlRelationDialectRuntime } from "./relation-runtime-auth.js"; +import { + isSqlSourceSnapshot, + type SqlSourceSnapshot, +} from "./source.js"; +import { + isSqlStatementSlotSnapshot, + type SqlStatementSlot, +} from "./statement-index.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, + SqlTextRange, +} from "./types.js"; + +export const MAX_COLUMN_QUERY_RELATIONS = 256; + +export interface SqlColumnQueryRelation { + readonly alias: SqlIdentifierComponent | null; + readonly path: SqlIdentifierPath; + readonly range: SqlTextRange; +} + +export type SqlColumnQuerySiteIssue = + | "derived-relation" + | "incomplete-relation" + | "nested-query" + | "opaque-template-context" + | "table-function"; + +export type SqlColumnQuerySiteResult = + | { + readonly status: "inactive"; + readonly reason: + | "cursor-in-comment" + | "cursor-in-string" + | "cursor-in-embedded-region" + | "not-column-position" + | "not-select-query"; + } + | { + readonly status: "unavailable"; + readonly reason: + | "ambiguous-query-site" + | "opaque-statement" + | "resource-limit"; + } + | { + readonly status: "ready"; + readonly coverage: "complete" | "partial"; + readonly issues: readonly SqlColumnQuerySiteIssue[]; + readonly prefix: SqlIdentifierComponent; + readonly qualifier: SqlIdentifierPath; + readonly relations: readonly SqlColumnQueryRelation[]; + readonly replacementRange: SqlTextRange; + }; + +interface Token extends BoundedSqlLexeme { + readonly depth: number; +} + +type Clause = + | "from" + | "group" + | "having" + | "join-condition" + | "limit" + | "order" + | "qualify" + | "select-list" + | "where"; + +const RELATION_END_WORDS: ReadonlySet = new Set([ + "cross", + "except", + "fetch", + "full", + "group", + "having", + "inner", + "intersect", + "join", + "left", + "limit", + "natural", + "offset", + "on", + "order", + "qualify", + "right", + "union", + "using", + "where", + "window", +]); + +function inactive( + reason: Extract< + SqlColumnQuerySiteResult, + { status: "inactive" } + >["reason"], +): SqlColumnQuerySiteResult { + return Object.freeze({ reason, status: "inactive" }); +} + +function unavailable( + reason: Extract< + SqlColumnQuerySiteResult, + { status: "unavailable" } + >["reason"], +): SqlColumnQuerySiteResult { + return Object.freeze({ reason, status: "unavailable" }); +} + +function word( + source: SqlSourceSnapshot, + token: Token | undefined, +): string | null { + return token?.kind === "word" + ? source.analysisText.slice(token.from, token.to).toLowerCase() + : null; +} + +function isIdentifier(token: Token | undefined): boolean { + return token?.kind === "word" || + token?.kind === "quoted-identifier"; +} + +function punctuation( + source: SqlSourceSnapshot, + token: Token | undefined, + expected: string, +): boolean { + return token?.kind === "punctuation" && + source.analysisText.slice(token.from, token.to) === expected; +} + +function tokenize( + source: SqlSourceSnapshot, + slot: Extract, + dialect: SqlRelationDialectRuntime, +): readonly Token[] | null { + const lexer = new BoundedSqlLexer( + source, + slot.source.from, + slot.source.to, + dialect.querySite.lexicalProfile, + ); + const tokens: Token[] = []; + let depth = 0; + for (let lexeme = lexer.next(); lexeme; lexeme = lexer.next()) { + if (punctuation(source, { ...lexeme, depth }, ")")) { + depth = Math.max(0, depth - 1); + } + tokens.push(Object.freeze({ ...lexeme, depth })); + if (punctuation(source, { ...lexeme, depth }, "(")) { + depth += 1; + } + } + return lexer.resource === null ? Object.freeze(tokens) : null; +} + +function decodePath( + source: SqlSourceSnapshot, + dialect: SqlRelationDialectRuntime, + from: number, + to: number, +): SqlIdentifierPath | null { + const raw = source.analysisText.slice(from, to); + const decoded = dialect.querySite.decodeRelationPath(raw, raw.length); + if ( + decoded.status !== "decoded" || + decoded.prefix.value.length === 0 + ) { + return null; + } + return Object.freeze([ + ...decoded.qualifier, + decoded.prefix, + ]); +} + +function parseNamedRelation( + source: SqlSourceSnapshot, + dialect: SqlRelationDialectRuntime, + tokens: readonly Token[], + start: number, + depth: number, +): { + readonly next: number; + readonly relation: SqlColumnQueryRelation | null; + readonly issue: SqlColumnQuerySiteIssue | null; +} { + const first = tokens[start]; + if (!isIdentifier(first) || first?.depth !== depth) { + return { + issue: punctuation(source, first, "(") + ? "derived-relation" + : "incomplete-relation", + next: start + 1, + relation: null, + }; + } + let end = start + 1; + while ( + punctuation(source, tokens[end], ".") && + tokens[end]?.depth === depth && + isIdentifier(tokens[end + 1]) && + tokens[end + 1]?.depth === depth + ) { + end += 2; + } + const lastPathToken = tokens[end - 1] ?? first; + const path = decodePath( + source, + dialect, + first.from, + lastPathToken.to, + ); + if (path === null) { + return { + issue: "incomplete-relation", + next: end, + relation: null, + }; + } + if (punctuation(source, tokens[end], "(")) { + return { + issue: "table-function", + next: end + 1, + relation: null, + }; + } + let alias: SqlIdentifierComponent | null = null; + const maybeAs = word(source, tokens[end]); + if (maybeAs === "as") end += 1; + const aliasToken = tokens[end]; + const aliasWord = aliasToken ? word(source, aliasToken) : null; + if ( + isIdentifier(aliasToken) && + aliasToken?.depth === depth && + (maybeAs === "as" || + aliasWord === null || + !RELATION_END_WORDS.has(aliasWord)) + ) { + const aliasPath = decodePath( + source, + dialect, + aliasToken.from, + aliasToken.to, + ); + if (aliasPath?.length === 1) { + alias = aliasPath[0] ?? null; + end += 1; + } + } + return { + issue: null, + next: end, + relation: Object.freeze({ + alias, + path, + range: Object.freeze({ + from: first.from, + to: lastPathToken.to, + }), + }), + }; +} + +function clauseAt( + source: SqlSourceSnapshot, + tokens: readonly Token[], + selectIndex: number, + depth: number, + position: number, +): Clause { + let clause: Clause = "select-list"; + for ( + let index = selectIndex + 1; + index < tokens.length && tokens[index]!.from < position; + index += 1 + ) { + const token = tokens[index]!; + if (token.depth !== depth) continue; + switch (word(source, token)) { + case "from": + clause = "from"; + break; + case "group": + clause = "group"; + break; + case "having": + clause = "having"; + break; + case "limit": + case "fetch": + case "offset": + clause = "limit"; + break; + case "on": + case "using": + clause = "join-condition"; + break; + case "order": + clause = "order"; + break; + case "qualify": + clause = "qualify"; + break; + case "where": + clause = "where"; + break; + } + } + return clause; +} + +function typedPath( + source: SqlSourceSnapshot, + tokens: readonly Token[], + position: number, + dialect: SqlRelationDialectRuntime, +): { + readonly prefix: SqlIdentifierComponent; + readonly qualifier: SqlIdentifierPath; + readonly replacementRange: SqlTextRange; +} | null { + let endIndex = tokens.findIndex((token) => + token.from < position && position <= token.to + ); + const endToken = tokens[endIndex]; + if ( + endIndex < 0 || + (!isIdentifier(endToken) && + !punctuation(source, endToken, ".")) + ) { + return Object.freeze({ + prefix: Object.freeze({ quoted: false, value: "" }), + qualifier: Object.freeze([]), + replacementRange: Object.freeze({ from: position, to: position }), + }); + } + let startIndex = endIndex; + let expectIdentifier = punctuation(source, endToken, "."); + while (startIndex > 0) { + const previous = tokens[startIndex - 1]; + if ( + expectIdentifier + ? !isIdentifier(previous) + : !punctuation(source, previous, ".") + ) { + break; + } + startIndex -= 1; + expectIdentifier = !expectIdentifier; + } + const from = tokens[startIndex]?.from ?? position; + const raw = source.analysisText.slice(from, position); + const decoded = dialect.querySite.decodeRelationPath(raw, raw.length); + if (decoded.status !== "decoded") return null; + return Object.freeze({ + prefix: decoded.prefix, + qualifier: decoded.qualifier, + replacementRange: Object.freeze({ + from: from + decoded.finalSegment.from, + to: from + decoded.finalSegment.to, + }), + }); +} + +function cursorTokenReason( + tokens: readonly Token[], + position: number, +): Extract< + SqlColumnQuerySiteResult, + { status: "inactive" } +>["reason"] | null { + const token = tokens.find((candidate) => + candidate.from <= position && position < candidate.to + ); + switch (token?.kind) { + case "barrier": + return "cursor-in-embedded-region"; + case "comment": + case "line-comment": + return "cursor-in-comment"; + case "string": + return "cursor-in-string"; + default: + return null; + } +} + +export function recognizeSqlColumnQuerySite( + source: SqlSourceSnapshot, + slot: SqlStatementSlot, + position: number, + dialect: SqlRelationDialectRuntime, +): SqlColumnQuerySiteResult { + if ( + !isSqlSourceSnapshot(source) || + !isSqlStatementSlotSnapshot(slot) || + !isSqlRelationDialectRuntime(dialect) || + !Number.isSafeInteger(position) || + position < 0 || + position > source.analysisText.length + ) { + return unavailable("ambiguous-query-site"); + } + if (slot.boundaryQuality === "opaque") { + return unavailable("opaque-statement"); + } + const tokens = tokenize(source, slot, dialect); + if (tokens === null) return unavailable("resource-limit"); + const cursorReason = cursorTokenReason(tokens, position); + if (cursorReason !== null) return inactive(cursorReason); + + let cursorDepth = 0; + for (const token of tokens) { + if (token.from >= position) break; + cursorDepth = token.depth; + if (punctuation(source, token, "(")) cursorDepth += 1; + } + let selectIndex = -1; + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]!; + if ( + token.from <= position && + token.depth <= cursorDepth && + word(source, token) === "select" + ) { + selectIndex = index; + } + } + if (selectIndex < 0) return inactive("not-select-query"); + const selectDepth = tokens[selectIndex]!.depth; + const clause = clauseAt( + source, + tokens, + selectIndex, + selectDepth, + position, + ); + if (clause === "from" || clause === "limit") { + return inactive("not-column-position"); + } + const path = typedPath(source, tokens, position, dialect); + if (path === null) { + return inactive("not-column-position"); + } + + const relations: SqlColumnQueryRelation[] = []; + const issues = new Set(); + let commaStartsRelation = false; + let inFrom = false; + for ( + let index = selectIndex + 1; + index < tokens.length; + index += 1 + ) { + const token = tokens[index]!; + if (token.depth < selectDepth) break; + if (token.depth !== selectDepth) { + if (word(source, token) === "select") issues.add("nested-query"); + continue; + } + const tokenWord = word(source, token); + if ( + tokenWord === "where" || + tokenWord === "group" || + tokenWord === "having" || + tokenWord === "qualify" || + tokenWord === "order" || + tokenWord === "limit" || + tokenWord === "union" || + tokenWord === "intersect" || + tokenWord === "except" + ) { + inFrom = false; + commaStartsRelation = false; + continue; + } + if (tokenWord === "on" || tokenWord === "using") { + commaStartsRelation = false; + continue; + } + const startsRelation = + tokenWord === "from" || + tokenWord === "join" || + (inFrom && + commaStartsRelation && + punctuation(source, token, ",")); + if (!startsRelation) continue; + inFrom = true; + const parsed = parseNamedRelation( + source, + dialect, + tokens, + index + 1, + selectDepth, + ); + if (parsed.issue !== null) issues.add(parsed.issue); + if (parsed.relation !== null) { + relations.push(parsed.relation); + if (relations.length > MAX_COLUMN_QUERY_RELATIONS) { + return unavailable("resource-limit"); + } + } + commaStartsRelation = true; + index = Math.max(index, parsed.next - 1); + } + const issueList = Object.freeze(Array.from(issues).sort()); + return Object.freeze({ + coverage: issueList.length === 0 ? "complete" : "partial", + issues: issueList, + prefix: path.prefix, + qualifier: path.qualifier, + relations: Object.freeze(relations), + replacementRange: path.replacementRange, + status: "ready", + }); +} diff --git a/src/vnext/index.ts b/src/vnext/index.ts index 5a98211..6753adb 100644 --- a/src/vnext/index.ts +++ b/src/vnext/index.ts @@ -5,6 +5,17 @@ export { duckdbDialect, postgresDialect, } from "./session.js"; +export type { + SqlColumnCatalogBatchRequest, + SqlColumnCatalogBatchResponse, + SqlColumnCatalogColumn, + SqlColumnCatalogCoverage, + SqlColumnCatalogProvider, + SqlColumnCatalogProvenance, + SqlColumnCatalogRelationReference, + SqlColumnCatalogRelationResult, + SqlColumnCatalogResolvedColumn, +} from "./column-catalog-types.js"; export type { OpenSqlDocument, SqlCatalogContext, @@ -53,6 +64,9 @@ export type { SqlCatalogSearchRequest, SqlCatalogSearchResponse, SqlCompletionCancellationReason, + SqlColumnCatalogProviderReport, + SqlColumnCompletionProvenance, + SqlCompletionProviderReport, SqlCompletionIssue, SqlCompletionRequest, SqlCompletionRefreshToken, diff --git a/src/vnext/local-relation-site.ts b/src/vnext/local-relation-site.ts index 5199eb0..4fa2c5e 100644 --- a/src/vnext/local-relation-site.ts +++ b/src/vnext/local-relation-site.ts @@ -4,6 +4,10 @@ import { type SqlCteVisibility, visibleSqlCtesAt, } from "./cte-layout.js"; +import { + recognizeSqlColumnQuerySite, + type SqlColumnQuerySiteResult, +} from "./column-query-site.js"; import { recognizeSqlRelationQuerySiteWithEntrypoints, type SqlQuerySiteResult, @@ -198,3 +202,31 @@ export function analyzeSqlLocalRelationSite( status: "ready", }); } + +export function analyzeSqlLocalColumnSite( + statement: SqlLocalRelationStatement, + position: number, +): SqlColumnQuerySiteResult { + if ( + statement === null || + typeof statement !== "object" + ) { + return Object.freeze({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + } + const context = localRelationStatements.get(statement); + if (!context) { + return Object.freeze({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + } + return recognizeSqlColumnQuerySite( + context.source, + context.slot, + position, + context.dialect, + ); +} diff --git a/src/vnext/relation-completion-types.ts b/src/vnext/relation-completion-types.ts index ce950fd..0dcccaf 100644 --- a/src/vnext/relation-completion-types.ts +++ b/src/vnext/relation-completion-types.ts @@ -249,6 +249,15 @@ export interface SqlCatalogCompletionProvenance { readonly entityId: string; } +export interface SqlColumnCompletionProvenance { + readonly kind: "column-catalog"; + readonly providerId: string; + readonly scope: string; + readonly epoch: SqlCatalogEpoch; + readonly relationEntityId: string; + readonly columnEntityId: string; +} + interface SqlCompletionItemBase { readonly label: string; readonly edit: SqlTextChange; @@ -265,6 +274,12 @@ export type SqlCompletionItem = readonly kind: "relation"; readonly relationKind: SqlCatalogRelationKind; readonly provenance: SqlCatalogCompletionProvenance; + }) + | (SqlCompletionItemBase & { + readonly dataType?: string; + readonly kind: "column"; + readonly provenance: SqlColumnCompletionProvenance; + readonly relationRequestKey: string; }); export type SqlCompletionIssue = @@ -281,7 +296,12 @@ export type SqlCompletionIssue = | "catalog-overloaded" | "catalog-queue-timeout" | "catalog-timeout" + | "column-catalog-failed" + | "column-catalog-loading" + | "column-catalog-malformed" + | "column-catalog-partial" | "cte-scope-uncertainty" + | "query-binding-partial" | "query-site-recovery" | "opaque-template-context" | "recursive-cte-uncertainty" @@ -345,6 +365,38 @@ export type SqlCatalogProviderReport = readonly reason: SqlCatalogProviderUnavailableReason; }); +export type SqlColumnCatalogProviderReport = + | { + readonly feature: "column-catalog"; + readonly outcome: "ready"; + readonly providerId: string; + readonly coverage: "complete" | "partial"; + } + | { + readonly feature: "column-catalog"; + readonly outcome: "loading"; + readonly providerId: string; + } + | { + readonly feature: "column-catalog"; + readonly outcome: "failed"; + readonly providerId: string; + } + | { + readonly feature: "column-catalog"; + readonly outcome: "unavailable"; + readonly providerId: string; + readonly reason: + | "disposed" + | "invalid-request" + | "malformed-response" + | "provider-failed"; + }; + +export type SqlCompletionProviderReport = + | SqlCatalogProviderReport + | SqlColumnCatalogProviderReport; + export interface SqlServiceFailure { readonly code: "internal"; readonly retryable: boolean; @@ -356,7 +408,7 @@ export type SqlCompletionResult = readonly revision: SqlRevision; readonly refreshToken: SqlCompletionRefreshToken | null; readonly value: SqlCompletionList; - readonly sources: readonly SqlCatalogProviderReport[]; + readonly sources: readonly SqlCompletionProviderReport[]; } | { readonly status: "unavailable"; diff --git a/src/vnext/relation-completion.ts b/src/vnext/relation-completion.ts index a0d2668..87ac3cb 100644 --- a/src/vnext/relation-completion.ts +++ b/src/vnext/relation-completion.ts @@ -72,7 +72,7 @@ interface RankedCteItem { interface RankedCatalogItem { readonly completionPathLength: number; readonly item: Exclude< - SqlCompletionItem, + Extract, { readonly relationKind: "cte" } >; readonly label: string; diff --git a/src/vnext/session.ts b/src/vnext/session.ts index bb5359d..cafecb2 100644 --- a/src/vnext/session.ts +++ b/src/vnext/session.ts @@ -10,6 +10,16 @@ import type { SqlTextChange, SqlTextRange, } from "./types.js"; +import { + createSqlColumnCatalogBatchCoordinator, + type SqlColumnCatalogBatchCoordinator, + type SqlColumnCatalogBatchOwner, + type SqlColumnCatalogBatchTicket, +} from "./column-catalog-batch-coordinator.js"; +import { + composeSqlColumnCompletion, + prepareSqlColumnCatalogRelations, +} from "./column-completion.js"; import { buildSqlStatementIndex, findSqlStatementSlot, @@ -34,6 +44,7 @@ import { type SqlCatalogSearchWorkTicket, } from "./relation-catalog-search-work.js"; import { + analyzeSqlLocalColumnSite, analyzeSqlLocalRelationSite, prepareSqlLocalRelationStatement, type SqlLocalRelationStatementPreparation, @@ -104,7 +115,10 @@ interface ResolvedCatalogContext { interface CompletionRequestState { cancelReason: "caller" | "disposed" | "superseded" | null; readonly revision: SqlRevision; - ticket: SqlCatalogSearchWorkTicket | null; + ticket: + | SqlCatalogSearchWorkTicket + | SqlColumnCatalogBatchTicket + | null; readonly token: SqlCompletionRefreshToken; } @@ -937,6 +951,7 @@ export class DefaultSqlDocumentSession { readonly #catalogCoordinator: SqlCatalogSearchWorkCoordinator | null; readonly #catalogResponseBudgetMs: number; + readonly #columnCoordinator: SqlColumnCatalogBatchCoordinator | null; readonly #dialects: ReadonlyMap; readonly #onDispose: () => void; readonly #listeners = new Set(); @@ -944,6 +959,9 @@ export class DefaultSqlDocumentSession #catalogOwner: SqlCatalogSearchWorkOwner | null = null; #catalogOwnerDialect: SqlRelationDialectRuntime | null = null; #catalogOwnerScope: string | null = null; + #columnOwner: SqlColumnCatalogBatchOwner | null = null; + #columnOwnerDialect: SqlRelationDialectRuntime | null = null; + #columnOwnerScope: string | null = null; #disposed = false; #localRelationStatementCache: | LocalRelationStatementCache @@ -960,10 +978,12 @@ export class DefaultSqlDocumentSession context: Context, dialects: ReadonlyMap, catalogCoordinator: SqlCatalogSearchWorkCoordinator | null, + columnCoordinator: SqlColumnCatalogBatchCoordinator | null, completion: CompletionConfiguration, onDispose: () => void, ) { this.#catalogCoordinator = catalogCoordinator; + this.#columnCoordinator = columnCoordinator; this.#catalogResponseBudgetMs = completion.catalogResponseBudgetMs; this.#dialects = dialects; @@ -984,6 +1004,7 @@ export class DefaultSqlDocumentSession sourceSequence, }); this.#replaceCatalogOwner(); + this.#replaceColumnOwner(); } get revision(): SqlRevision { @@ -1240,6 +1261,34 @@ export class DefaultSqlDocumentSession } } + #replaceColumnOwner(): void { + const catalog = resolveCatalogContext(this.#snapshot.context); + const dialect = this.#snapshot.dialect.relationDialect; + if ( + this.#columnOwner && + catalog && + this.#columnOwnerScope === catalog.scope && + this.#columnOwnerDialect === dialect + ) { + return; + } + this.#columnOwner?.dispose(); + this.#columnOwner = null; + this.#columnOwnerDialect = null; + this.#columnOwnerScope = null; + if (!catalog || !this.#columnCoordinator || this.#disposed) { + return; + } + const prepared = this.#columnCoordinator.prepareOwner({ + dialectId: dialect.id, + scope: catalog.scope, + }); + if (prepared.status !== "prepared") return; + this.#columnOwner = prepared.owner; + this.#columnOwnerDialect = dialect; + this.#columnOwnerScope = catalog.scope; + } + readonly onDidChange = ( listener: (event: SqlSessionChangeEvent) => void, ): SqlDisposable => { @@ -1539,6 +1588,125 @@ export class DefaultSqlDocumentSession position, ); if (localSite.status !== "ready") { + const columnSite = analyzeSqlLocalColumnSite( + prepared.statement, + position, + ); + if ( + localSite.status === "inactive" && + columnSite.status === "ready" + ) { + const catalog = resolveCatalogContext(snapshot.context); + const columnOwner = this.#columnOwner; + const columnCoordinator = this.#columnCoordinator; + const preparedRelations = prepareSqlColumnCatalogRelations( + columnSite, + snapshot.dialect.relationDialect, + ); + if (preparedRelations.references.length === 0) { + cancelPrevious(); + return Object.freeze({ + reason: unavailableCompletionReason(localSite), + retryable: false, + revision: snapshot.revision, + status: "unavailable", + }); + } + if (!catalog || !columnOwner || !columnCoordinator) { + cancelPrevious(); + return Object.freeze({ + reason: "unsupported-query-site", + retryable: false, + revision: snapshot.revision, + status: "unavailable", + }); + } + if ( + this.#activeCompletion !== active || + active.cancelReason !== null || + snapshot.revision !== this.#snapshot.revision + ) { + return completionCancellation( + snapshot.revision, + completionCancellationReason(active), + ); + } + const ticket = columnOwner.request({ + expectedEpoch: null, + relations: preparedRelations.references, + searchPaths: catalog.searchPaths, + }); + active.ticket = ticket; + let responseTimer: + | ReturnType + | undefined; + const raced = await Promise.race([ + ticket.result.then((outcome) => + Object.freeze({ + kind: "outcome" as const, + outcome, + }), + ), + new Promise<{ readonly kind: "timeout" }>((resolve) => { + responseTimer = setTimeout( + () => resolve(Object.freeze({ kind: "timeout" })), + this.#catalogResponseBudgetMs, + ); + }), + ]); + clearTimeout(responseTimer); + if ( + this.#activeCompletion !== active || + active.cancelReason !== null || + snapshot.revision !== this.#snapshot.revision + ) { + ticket.cancel(); + return completionCancellation( + snapshot.revision, + completionCancellationReason(active), + ); + } + if (raced.kind === "timeout") { + ticket.cancel(); + return Object.freeze({ + refreshToken: null, + revision: snapshot.revision, + sources: Object.freeze([Object.freeze({ + feature: "column-catalog" as const, + outcome: "loading" as const, + providerId: columnCoordinator.providerId, + })]), + status: "ready", + value: Object.freeze({ + isIncomplete: true, + issues: Object.freeze([Object.freeze({ + reason: "column-catalog-loading" as const, + })] as const), + items: Object.freeze([]), + }), + }); + } + const composition = composeSqlColumnCompletion({ + dialect: snapshot.dialect.relationDialect, + outcome: raced.outcome, + prepared: preparedRelations, + providerId: columnCoordinator.providerId, + site: columnSite, + }); + if (!composition) { + return completionCancellation( + snapshot.revision, + completionCancellationReason(active), + ); + } + return Object.freeze({ + refreshToken: null, + revision: snapshot.revision, + sources: composition.sources, + status: "ready", + value: composition.value, + }); + } cancelPrevious(); return Object.freeze({ reason: unavailableCompletionReason(localSite), @@ -2121,6 +2289,7 @@ export class DefaultSqlDocumentSession refreshIntent?.ticket?.cancel(); } this.#replaceCatalogOwner(); + this.#replaceColumnOwner(); return this.#snapshot.revision; } @@ -2136,9 +2305,11 @@ export class DefaultSqlDocumentSession const activeCompletion = this.#activeCompletion; const refreshIntent = this.#refreshIntent; const catalogOwner = this.#catalogOwner; + const columnOwner = this.#columnOwner; this.#activeCompletion = null; this.#refreshIntent = null; this.#catalogOwner = null; + this.#columnOwner = null; this.#clearSoftRefreshIntentTimer(); this.#clearTerminalIntent(); this.#listeners.clear(); @@ -2162,6 +2333,7 @@ export class DefaultSqlDocumentSession refreshIntent?.ticket?.cancel(); } catalogOwner?.dispose(); + columnOwner?.dispose(); this.#onDispose(); }; } @@ -2170,6 +2342,7 @@ export class DefaultSqlLanguageService implements SqlLanguageService { readonly #catalogCoordinator: SqlCatalogSearchWorkCoordinator | null; + readonly #columnCoordinator: SqlColumnCatalogBatchCoordinator | null; readonly #completion: CompletionConfiguration; readonly #dialects: ReadonlyMap; readonly #sessions = new Set>(); @@ -2303,6 +2476,27 @@ export class DefaultSqlLanguageService } this.#catalogCoordinator = coordinator.coordinator; } + + const columns = readOwnDataProperty( + options, + "columns", + "invalid-service-options", + "SQL language service options", + ); + if (!columns.found || columns.value === undefined) { + this.#columnCoordinator = null; + } else { + const coordinator = createSqlColumnCatalogBatchCoordinator({ + provider: columns.value, + }); + if (coordinator.status !== "created") { + throw new SqlSessionError( + "invalid-service-options", + "SQL column catalog provider is invalid", + ); + } + this.#columnCoordinator = coordinator.coordinator; + } } catch (error) { if (error instanceof SqlSessionError) { throw error; @@ -2378,6 +2572,7 @@ export class DefaultSqlLanguageService context, this.#dialects, this.#catalogCoordinator, + this.#columnCoordinator, this.#completion, () => { this.#sessions.delete(session); @@ -2419,6 +2614,7 @@ export class DefaultSqlLanguageService } this.#sessions.clear(); this.#catalogCoordinator?.dispose(); + this.#columnCoordinator?.dispose(); }; } diff --git a/src/vnext/types.ts b/src/vnext/types.ts index 075c403..d2055cb 100644 --- a/src/vnext/types.ts +++ b/src/vnext/types.ts @@ -162,6 +162,7 @@ export interface SqlLanguageService { export interface SqlLanguageServiceOptions { readonly catalog?: SqlRelationCatalogProvider | undefined; + readonly columns?: SqlColumnCatalogProvider | undefined; readonly completion?: { readonly catalogResponseBudgetMs?: number | undefined; } | undefined; @@ -192,6 +193,9 @@ export class SqlSessionError extends Error { this.code = code; } } +import type { + SqlColumnCatalogProvider, +} from "./column-catalog-types.js"; import type { SqlCompletionRequest, SqlCompletionTask, From c20bcf6158260c7819dcc11c4f36d7936da86f66 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:52:45 +0800 Subject: [PATCH 09/25] test(vnext): expect worker query bindings --- src/vnext/browser_tests/node-sql-parser-browser-worker.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/vnext/browser_tests/node-sql-parser-browser-worker.test.ts b/src/vnext/browser_tests/node-sql-parser-browser-worker.test.ts index 714a853..968dcf6 100644 --- a/src/vnext/browser_tests/node-sql-parser-browser-worker.test.ts +++ b/src/vnext/browser_tests/node-sql-parser-browser-worker.test.ts @@ -151,6 +151,7 @@ test( expect(firstPostgresql).toStrictEqual({ kind: "parsed", protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, + queryBindings: expect.any(Object), requestId: 1, statementKind: "query", }); @@ -166,6 +167,7 @@ test( ).resolves.toStrictEqual({ kind: "parsed", protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, + queryBindings: expect.any(Object), requestId: 2, statementKind: "query", }); @@ -180,6 +182,7 @@ test( ).resolves.toStrictEqual({ kind: "parsed", protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, + queryBindings: expect.any(Object), requestId: 3, statementKind: "query", }); From 345ea3bd655e32a3a55be2f34901785033259974 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:53:21 +0800 Subject: [PATCH 10/25] feat(vnext): add namespace catalog completion --- docs/vnext/namespace-catalog-authority.md | 37 ++ .../namespace-catalog-boundary.test.ts | 265 +++++++++ .../namespace-catalog-coordinator.bench.ts | 69 +++ .../namespace-catalog-coordinator.test.ts | 521 ++++++++++++++++ src/vnext/namespace-catalog-boundary.ts | 558 ++++++++++++++++++ src/vnext/namespace-catalog-coordinator.ts | 488 +++++++++++++++ src/vnext/namespace-catalog-types.ts | 96 +++ src/vnext/namespace-completion.ts | 293 +++++++++ 8 files changed, 2327 insertions(+) create mode 100644 docs/vnext/namespace-catalog-authority.md create mode 100644 src/vnext/__tests__/namespace-catalog-boundary.test.ts create mode 100644 src/vnext/__tests__/namespace-catalog-coordinator.bench.ts create mode 100644 src/vnext/__tests__/namespace-catalog-coordinator.test.ts create mode 100644 src/vnext/namespace-catalog-boundary.ts create mode 100644 src/vnext/namespace-catalog-coordinator.ts create mode 100644 src/vnext/namespace-catalog-types.ts create mode 100644 src/vnext/namespace-completion.ts diff --git a/docs/vnext/namespace-catalog-authority.md b/docs/vnext/namespace-catalog-authority.md new file mode 100644 index 0000000..e3ef97d --- /dev/null +++ b/docs/vnext/namespace-catalog-authority.md @@ -0,0 +1,37 @@ +# vNext Namespace Catalog Authority + +Status: integration-ready internal vertical slice + +Namespace completion searches catalog, schema, project, and dataset containers. +One query site produces one bounded provider request containing its qualifier, +prefix, search paths, result limit, dialect, scope, and expected catalog epoch. +The API does not recursively walk a provider tree or issue one request per +container. + +Each result has a stable container entity ID, a canonical role-tagged identifier +path, provider-rendered insertion text, match quality, and immutable provenance. +The hostile boundary rejects accessors, sparse arrays, extra properties, +oversized data, invalid epochs, and conflicting duplicate identities. Identical +duplicate identities are collapsed and results receive deterministic code-unit +ordering. + +The composer applies the dialect's prefix matcher, deduplicates by authority +identity, and produces deterministic completion edits. Unknown or throwing +prefix comparison is reported as incomplete instead of guessing. + +## Epoch, cache, and lifecycle + +A cold request uses `expectedEpoch: null`; the returned epoch becomes the +owner's observed epoch. Complete ready searches are held in a bounded LRU keyed +by provider instance, scope, dialect, epoch, query site, search paths, and limit. +Partial, loading, and failed responses remain visible but are never cached. + +A newer request supersedes and aborts prior work for the same owner. Explicit +cancellation and owner/coordinator disposal abort pending work. Provider throws, +rejections, malformed data, and late settlements are contained. + +Session integration should prepare one owner for each live scope/dialect and +dispose it when catalog authority changes. Query-site integration calls +`prepareSqlNamespaceCatalogSearch`, submits the result through the owner, and +passes the outcome plus the site's replacement range and dialect prefix matcher +to `composeSqlNamespaceCompletion`. diff --git a/src/vnext/__tests__/namespace-catalog-boundary.test.ts b/src/vnext/__tests__/namespace-catalog-boundary.test.ts new file mode 100644 index 0000000..4bee715 --- /dev/null +++ b/src/vnext/__tests__/namespace-catalog-boundary.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it, vi } from "vitest"; +import { + captureSqlNamespaceCatalogProvider, + createSqlNamespaceCatalogSearchRequest, + decodeSqlNamespaceCatalogSearchResponse, + MAX_NAMESPACE_RESULTS, + resolveSqlNamespaceCatalogProvider, +} from "../namespace-catalog-boundary.js"; + +const epoch = Object.freeze({ generation: 1, token: "one" }); + +function request(expectedEpoch = epoch) { + const result = createSqlNamespaceCatalogSearchRequest({ + dialectId: "duckdb", + expectedEpoch, + limit: 10, + prefix: { quoted: false, value: "ma" }, + qualifier: [{ quoted: false, value: "memory" }], + scope: "connection", + searchPaths: [[{ quoted: false, value: "main" }]], + }); + if (result.status !== "accepted") throw new Error("request"); + return result.value; +} + +function captured(search: unknown = vi.fn()) { + const result = captureSqlNamespaceCatalogProvider({ + id: "namespaces", + search, + }); + if (result.status !== "accepted") throw new Error("provider"); + return result.value; +} + +function container( + id: string, + role: "catalog" | "dataset" | "project" | "schema", + value: string, +) { + return { + canonicalPath: [{ + quoted: false, + role, + value, + }], + containerEntityId: id, + detail: `${role} detail`, + insertText: value, + matchQuality: "exact", + }; +} + +describe("namespace catalog boundary", () => { + it("captures provider methods without exposing a receiver", () => { + const receivers: unknown[] = []; + const search = function (this: unknown) { + receivers.push(this); + }; + const handle = captured(search); + const context = resolveSqlNamespaceCatalogProvider(handle); + expect(context?.id).toBe("namespaces"); + context?.search(request(), new AbortController().signal); + expect(receivers).toEqual([undefined]); + expect(Object.isFrozen(handle)).toBe(true); + }); + + it("rejects hostile or invalid providers", () => { + const accessor = {}; + Object.defineProperty(accessor, "id", { get: vi.fn() }); + expect(captureSqlNamespaceCatalogProvider(accessor).status) + .toBe("malformed"); + expect(captureSqlNamespaceCatalogProvider(null).status) + .toBe("malformed"); + expect(captureSqlNamespaceCatalogProvider({ + id: "", + search() {}, + }).status).toBe("malformed"); + expect(captureSqlNamespaceCatalogProvider({ + id: "x", + search: 1, + }).status).toBe("malformed"); + expect(captureSqlNamespaceCatalogProvider({ + extra: true, + id: "x", + search() {}, + }).status).toBe("malformed"); + expect(captureSqlNamespaceCatalogProvider( + new Proxy({}, { ownKeys: () => { throw new Error("trap"); } }), + ).status).toBe("malformed"); + }); + + it("normalizes and freezes one bounded query-site search", () => { + const value = request(); + expect(value).toEqual({ + dialectId: "duckdb", + expectedEpoch: epoch, + limit: 10, + prefix: { quoted: false, value: "ma" }, + qualifier: [{ quoted: false, value: "memory" }], + scope: "connection", + searchPaths: [[{ quoted: false, value: "main" }]], + }); + expect(Object.isFrozen(value)).toBe(true); + expect(Object.isFrozen(value.prefix)).toBe(true); + expect(Object.isFrozen(value.qualifier)).toBe(true); + expect(Object.isFrozen(value.searchPaths[0])).toBe(true); + expect(createSqlNamespaceCatalogSearchRequest({ + ...value, + expectedEpoch: null, + }).status).toBe("accepted"); + }); + + it("rejects malformed, sparse, accessor, and oversized requests", () => { + const valid = request(); + const sparse: unknown[] = []; + sparse.length = 1; + const cases: unknown[] = [ + null, + { ...valid, extra: true }, + { ...valid, dialectId: "" }, + { ...valid, scope: "" }, + { ...valid, expectedEpoch: undefined }, + { ...valid, expectedEpoch: { generation: -1, token: "x" } }, + { ...valid, expectedEpoch: { generation: 1.5, token: "x" } }, + { ...valid, expectedEpoch: { generation: 1, token: "" } }, + { ...valid, limit: 0 }, + { ...valid, limit: MAX_NAMESPACE_RESULTS + 1 }, + { ...valid, limit: 1.5 }, + { ...valid, prefix: { quoted: "no", value: "x" } }, + { ...valid, prefix: { quoted: false, value: "x", extra: 1 } }, + { ...valid, qualifier: sparse }, + { ...valid, searchPaths: sparse }, + { ...valid, qualifier: [{ quoted: false, value: "x".repeat(257) }] }, + { ...valid, searchPaths: Array.from({ length: 33 }, () => []) }, + ]; + const accessor = { ...valid }; + Object.defineProperty(accessor, "prefix", { get: vi.fn() }); + cases.push(accessor); + for (const value of cases) { + expect(createSqlNamespaceCatalogSearchRequest(value).status) + .toBe("malformed"); + } + }); + + it("decodes, deduplicates, orders, and freezes all container roles", () => { + const provider = captured(); + const decoded = decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + { + containers: [ + container("schema", "schema", "main"), + { ...container("schema", "schema", "main") }, + container("project", "project", "alpha"), + container("dataset", "dataset", "events"), + container("catalog", "catalog", "memory"), + ], + coverage: "partial", + epoch, + status: "ready", + }, + ); + expect(decoded.status).toBe("accepted"); + if (decoded.status !== "accepted" || + decoded.value.status !== "ready") return; + expect(decoded.value.containers.map((value) => + value.canonicalPath[0]?.role + )).toEqual(["catalog", "dataset", "project", "schema"]); + expect(decoded.value.containers[0]?.provenance).toEqual({ + containerEntityId: "catalog", + epoch, + providerId: "namespaces", + scope: "connection", + }); + expect(Object.isFrozen(decoded.value)).toBe(true); + expect(Object.isFrozen(decoded.value.containers)).toBe(true); + expect(Object.isFrozen( + decoded.value.containers[0]?.canonicalPath, + )).toBe(true); + }); + + it("decodes loading and normalized failures", () => { + const provider = captured(); + expect(decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + { epoch, status: "loading" }, + )).toMatchObject({ + status: "accepted", + value: { status: "loading" }, + }); + expect(decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + { + code: "rate-limited", + epoch, + retry: "next-request", + status: "failed", + }, + )).toMatchObject({ + status: "accepted", + value: { + code: "rate-limited", + retry: "next-request", + status: "failed", + }, + }); + }); + + it("rejects malformed, conflicting, stale, and oversized responses", () => { + const provider = captured(); + const valid = { + containers: [container("one", "schema", "main")], + coverage: "complete", + epoch, + status: "ready", + }; + const conflicting = { + ...valid, + containers: [ + container("same", "schema", "main"), + container("same", "schema", "other"), + ], + }; + const malformedContainers = [ + { ...container("x", "schema", "x"), canonicalPath: [] }, + { ...container("x", "schema", "x"), canonicalPath: [{ + quoted: false, + role: "table", + value: "x", + }] }, + { ...container("x", "schema", "x"), containerEntityId: "" }, + { ...container("x", "schema", "x"), insertText: "" }, + { ...container("x", "schema", "x"), matchQuality: "bad" }, + { ...container("x", "schema", "x"), detail: 1 }, + ]; + const cases: unknown[] = [ + null, + { ...valid, extra: true }, + { ...valid, epoch: { generation: 2, token: "two" } }, + { ...valid, coverage: "unknown" }, + { ...valid, containers: Array.from( + { length: 11 }, + (_, index) => container(String(index), "schema", String(index)), + ) }, + conflicting, + { epoch, extra: true, status: "loading" }, + { code: "bad", epoch, retry: "never", status: "failed" }, + { code: "unknown", epoch, retry: "bad", status: "failed" }, + { code: "unknown", epoch, retry: "never", status: "failed", x: 1 }, + ...malformedContainers.map((value) => ({ + ...valid, + containers: [value], + })), + ]; + for (const value of cases) { + expect(decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + value, + ).status).toBe("malformed"); + } + }); +}); diff --git a/src/vnext/__tests__/namespace-catalog-coordinator.bench.ts b/src/vnext/__tests__/namespace-catalog-coordinator.bench.ts new file mode 100644 index 0000000..c42dbfd --- /dev/null +++ b/src/vnext/__tests__/namespace-catalog-coordinator.bench.ts @@ -0,0 +1,69 @@ +import { bench, describe } from "vitest"; +import { + createSqlNamespaceCatalogCoordinator, +} from "../namespace-catalog-coordinator.js"; +import type { + SqlNamespaceCatalogSearchRequest, +} from "../namespace-catalog-types.js"; + +const epoch = Object.freeze({ generation: 1, token: "bench" }); + +function response(request: SqlNamespaceCatalogSearchRequest) { + return { + containers: Array.from({ length: 100 }, (_, index) => ({ + canonicalPath: [ + { quoted: false, role: "catalog", value: "memory" }, + { quoted: false, role: "schema", value: `schema_${index}` }, + ], + containerEntityId: `schema:${index}`, + insertText: `schema_${index}`, + matchQuality: "exact", + })), + coverage: "complete", + epoch: request.expectedEpoch ?? epoch, + status: "ready", + }; +} + +function owner() { + const created = createSqlNamespaceCatalogCoordinator({ + provider: { + id: "benchmark", + search: (request: SqlNamespaceCatalogSearchRequest) => + Promise.resolve(response(request)), + }, + }); + if (created.status !== "created") throw new Error("coordinator"); + const prepared = created.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "benchmark", + }); + if (prepared.status !== "prepared") throw new Error("owner"); + return prepared.owner; +} + +const search = Object.freeze({ + expectedEpoch: epoch, + limit: 100, + prefix: Object.freeze({ quoted: false, value: "schema" }), + qualifier: Object.freeze([ + Object.freeze({ quoted: false, value: "memory" }), + ]), + searchPaths: Object.freeze([]), +}); + +describe("namespace catalog coordinator", () => { + bench("cold 100-container search", async () => { + const current = owner(); + await current.request(search).result; + current.dispose(); + }); + + const warm = owner(); + const primed = warm.request(search).result; + + bench("warm 100-container cache lookup", async () => { + await primed; + await warm.request(search).result; + }); +}); diff --git a/src/vnext/__tests__/namespace-catalog-coordinator.test.ts b/src/vnext/__tests__/namespace-catalog-coordinator.test.ts new file mode 100644 index 0000000..7f1f062 --- /dev/null +++ b/src/vnext/__tests__/namespace-catalog-coordinator.test.ts @@ -0,0 +1,521 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createSqlNamespaceCatalogCoordinator, + type SqlNamespaceCatalogSearchOutcome, +} from "../namespace-catalog-coordinator.js"; +import { + composeSqlNamespaceCompletion, + prepareSqlNamespaceCatalogSearch, +} from "../namespace-completion.js"; +import type { + SqlNamespaceCatalogSearchRequest, +} from "../namespace-catalog-types.js"; +import type { SqlCatalogEpoch } from "../relation-completion-types.js"; + +const epoch: SqlCatalogEpoch = + Object.freeze({ generation: 1, token: "one" }); +const nextEpoch: SqlCatalogEpoch = + Object.freeze({ generation: 2, token: "two" }); + +function input( + prefix = "ma", + expectedEpoch: SqlCatalogEpoch | null = epoch, +) { + return { + expectedEpoch, + limit: 10, + prefix: { quoted: false, value: prefix }, + qualifier: [{ quoted: false, value: "memory" }], + searchPaths: [[{ quoted: false, value: "main" }]], + }; +} + +function response( + request: SqlNamespaceCatalogSearchRequest, + coverage: "complete" | "partial" = "complete", +) { + return { + containers: [{ + canonicalPath: [ + { quoted: false, role: "catalog", value: "memory" }, + { quoted: false, role: "schema", value: "main" }, + ], + containerEntityId: `schema:${request.prefix.value}`, + detail: "Schema", + insertText: "main", + matchQuality: "exact", + }], + coverage, + epoch: request.expectedEpoch ?? epoch, + status: "ready", + }; +} + +function setup( + search: ( + request: SqlNamespaceCatalogSearchRequest, + signal: AbortSignal, + ) => unknown, + maxCacheEntries = 16, +) { + const created = createSqlNamespaceCatalogCoordinator({ + maxCacheEntries, + provider: { id: "namespaces", search }, + }); + if (created.status !== "created") throw new Error("coordinator"); + const prepared = created.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "connection", + }); + if (prepared.status !== "prepared") throw new Error("owner"); + return { + coordinator: created.coordinator, + owner: prepared.owner, + }; +} + +function deferred() { + let resolve: (value: Value) => void = (): void => {}; + let reject: (reason?: unknown) => void = (): void => {}; + const promise = new Promise((resolve_, reject_) => { + resolve = resolve_; + reject = reject_; + }); + return { promise, reject, resolve }; +} + +describe("namespace catalog coordinator", () => { + it("performs exactly one bounded search for one query site", async () => { + const searches: SqlNamespaceCatalogSearchRequest[] = []; + const { coordinator, owner } = setup((request) => { + searches.push(request); + return response(request); + }); + const outcome = await owner.request(input()).result; + expect(searches).toHaveLength(1); + expect(searches[0]).toMatchObject({ + dialectId: "duckdb", + limit: 10, + prefix: { value: "ma" }, + qualifier: [{ value: "memory" }], + scope: "connection", + }); + expect(outcome).toMatchObject({ + providerId: "namespaces", + response: { status: "ready" }, + scope: "connection", + status: "usable", + }); + expect(Object.isFrozen(outcome)).toBe(true); + coordinator.dispose(); + }); + + it("observes cold epochs and caches complete searches", async () => { + const expected: Array = []; + const { owner } = setup((request) => { + expected.push(request.expectedEpoch); + return { ...response(request), epoch }; + }); + await owner.request(input("ma", null)).result; + const cached = await owner.request(input("ma", null)).result; + await owner.request(input("ma", nextEpoch)).result; + expect(expected).toEqual([null, nextEpoch]); + expect(cached.status).toBe("usable"); + }); + + it("does not cache partial, loading, or failed results", async () => { + let calls = 0; + const { owner } = setup((request) => { + calls += 1; + if (calls <= 2) return response(request, "partial"); + if (calls <= 4) { + return { epoch, status: "loading" }; + } + return { + code: "unavailable", + epoch, + retry: "next-request", + status: "failed", + }; + }); + await owner.request(input("partial")).result; + await owner.request(input("partial")).result; + await owner.request(input("loading")).result; + await owner.request(input("loading")).result; + await owner.request(input("failed")).result; + await owner.request(input("failed")).result; + expect(calls).toBe(6); + }); + + it("cancels and supersedes pending owner searches", async () => { + const pending: Array>> = []; + const signals: AbortSignal[] = []; + const { owner } = setup((_request, signal) => { + const work = deferred(); + pending.push(work); + signals.push(signal); + return work.promise; + }); + const first = owner.request(input("first")); + const second = owner.request(input("second")); + await expect(first.result).resolves.toEqual({ + status: "superseded", + }); + expect(signals[0]?.aborted).toBe(true); + second.cancel(); + second.cancel(); + await expect(second.result).resolves.toEqual({ + status: "cancelled", + }); + expect(signals[1]?.aborted).toBe(true); + pending[0]?.resolve({}); + pending[1]?.resolve({}); + await Promise.resolve(); + }); + + it("isolates owners and settles disposal", async () => { + const pending: Array>> = []; + const { coordinator, owner } = setup(() => { + const work = deferred(); + pending.push(work); + return work.promise; + }); + const other = coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "connection", + }); + if (other.status !== "prepared") throw new Error("second owner"); + const first = owner.request(input("first")); + const second = other.owner.request(input("second")); + owner.dispose(); + await expect(first.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + let secondSettled = false; + void second.result.then(() => { + secondSettled = true; + }); + await Promise.resolve(); + expect(secondSettled).toBe(false); + coordinator.dispose(); + await expect(second.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + expect(coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "connection", + })).toMatchObject({ + reason: "disposed", + status: "unavailable", + }); + expect(owner.request(input())).toBeDefined(); + }); + + it("contains provider failures and malformed settlements", async () => { + const thrown = setup(() => { + throw new Error("sync"); + }); + await expect(thrown.owner.request(input()).result).resolves + .toMatchObject({ + reason: "provider-failed", + status: "unavailable", + }); + const rejected = setup(() => Promise.reject(new Error("async"))); + await expect(rejected.owner.request(input()).result).resolves + .toMatchObject({ + reason: "provider-failed", + status: "unavailable", + }); + const malformed = setup(() => ({})); + await expect(malformed.owner.request(input()).result).resolves + .toMatchObject({ + reason: "malformed-response", + status: "unavailable", + }); + }); + + it("bounds the LRU and re-fetches evicted searches", async () => { + let calls = 0; + const { owner } = setup((request) => { + calls += 1; + return response(request); + }, 1); + await owner.request(input("one")).result; + await owner.request(input("two")).result; + await owner.request(input("one")).result; + expect(calls).toBe(3); + }); + + it("fails closed for hostile or invalid configuration and inputs", async () => { + expect(createSqlNamespaceCatalogCoordinator(null)).toMatchObject({ + reason: "invalid-provider", + status: "unavailable", + }); + expect(createSqlNamespaceCatalogCoordinator({ + maxCacheEntries: 0, + provider: { id: "x", search() {} }, + })).toMatchObject({ + reason: "invalid-options", + status: "unavailable", + }); + const created = createSqlNamespaceCatalogCoordinator({ + provider: { id: "x", search() {} }, + }); + if (created.status !== "created") throw new Error("created"); + expect(created.coordinator.prepareOwner({ + dialectId: "", + scope: "", + })).toMatchObject({ + reason: "invalid-options", + status: "unavailable", + }); + const prepared = created.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + }); + if (prepared.status !== "prepared") throw new Error("prepared"); + expect(await prepared.owner.request({ + ...input(), + limit: 0, + }).result).toMatchObject({ + reason: "invalid-request", + status: "unavailable", + }); + const hostile = {}; + Object.defineProperty(hostile, "expectedEpoch", { get: vi.fn() }); + const hostileTicket = Reflect.apply( + prepared.owner.request, + undefined, + [hostile], + ); + expect(await hostileTicket.result).toMatchObject({ + reason: "invalid-request", + status: "unavailable", + }); + }); +}); + +describe("namespace completion composer", () => { + const readyOutcome: Extract< + SqlNamespaceCatalogSearchOutcome, + { readonly status: "usable" } + > = { + providerId: "namespaces", + response: { + containers: [ + { + canonicalPath: [ + { quoted: false, role: "catalog", value: "memory" }, + { quoted: false, role: "schema", value: "main" }, + ], + containerEntityId: "main", + detail: "Schema", + insertText: "main", + matchQuality: "exact", + provenance: { + containerEntityId: "main", + epoch, + providerId: "namespaces", + scope: "connection", + }, + }, + { + canonicalPath: [ + { quoted: false, role: "project", value: "alpha" }, + { quoted: false, role: "dataset", value: "metrics" }, + ], + containerEntityId: "metrics", + insertText: "metrics", + matchQuality: "equivalent", + provenance: { + containerEntityId: "metrics", + epoch, + providerId: "namespaces", + scope: "connection", + }, + }, + ], + coverage: "complete", + epoch, + status: "ready", + }, + scope: "connection", + status: "usable", + }; + + function catalogResponse() { + if (readyOutcome.response.status !== "ready") { + throw new Error("ready response"); + } + return readyOutcome.response; + } + + it("prepares one immutable search from a query site", () => { + const prepared = prepareSqlNamespaceCatalogSearch({ + prefix: { quoted: false, value: "ma" }, + qualifier: [{ quoted: false, value: "memory" }], + replacementRange: { from: 7, to: 9 }, + }, null, [], 20); + expect(prepared).toEqual({ + expectedEpoch: null, + limit: 20, + prefix: { quoted: false, value: "ma" }, + qualifier: [{ quoted: false, value: "memory" }], + searchPaths: [], + }); + expect(Object.isFrozen(prepared)).toBe(true); + }); + + it("filters prefixes, deduplicates, orders, and uses provider edits", () => { + const catalog = catalogResponse(); + const main = catalog.containers[0]; + if (!main) throw new Error("main"); + const duplicated = { + ...readyOutcome, + response: { + ...catalog, + containers: [ + ...catalog.containers, + main, + { + ...main, + containerEntityId: "main-copy", + provenance: { + ...main.provenance, + containerEntityId: "main-copy", + }, + }, + ], + }, + }; + const all = composeSqlNamespaceCompletion({ + matchPrefix: (): "match" => "match", + outcome: duplicated, + prefix: { quoted: false, value: "" }, + providerId: "namespaces", + replacementRange: { from: 10, to: 12 }, + }); + expect(all?.value.items.map((value) => value.label)) + .toEqual(["main", "main", "metrics"]); + const composition = composeSqlNamespaceCompletion({ + matchPrefix: (candidate, prefix) => + candidate.value.startsWith(prefix.value) + ? "match" + : "no-match", + outcome: duplicated, + prefix: { quoted: false, value: "ma" }, + providerId: "namespaces", + replacementRange: { from: 10, to: 12 }, + }); + expect(composition).toMatchObject({ + source: { coverage: "complete", outcome: "ready" }, + value: { + isIncomplete: false, + items: [{ + edit: { from: 10, insert: "main", to: 12 }, + label: "main", + role: "schema", + }, { + edit: { from: 10, insert: "main", to: 12 }, + label: "main", + role: "schema", + }], + }, + }); + expect(Object.isFrozen(composition?.value.items)).toBe(true); + }); + + it("surfaces partial and uncertain prefix coverage", () => { + const catalog = catalogResponse(); + const composition = composeSqlNamespaceCompletion({ + matchPrefix: () => { + throw new Error("dialect"); + }, + outcome: { + ...readyOutcome, + response: { ...catalog, coverage: "partial" }, + }, + prefix: { quoted: false, value: "m" }, + providerId: "namespaces", + replacementRange: { from: 0, to: 1 }, + }); + expect(composition?.value).toMatchObject({ + isIncomplete: true, + issues: [ + "namespace-catalog-partial", + "namespace-prefix-uncertain", + ], + items: [], + }); + }); + + it("maps loading, failed, unavailable, and cancellation", () => { + const base = { + matchPrefix: (): "match" => "match", + prefix: { quoted: false, value: "" }, + providerId: "namespaces", + replacementRange: { from: 0, to: 0 }, + }; + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { + providerId: "namespaces", + response: { epoch, status: "loading" }, + scope: "connection", + status: "usable", + }, + })).toMatchObject({ + source: { outcome: "loading" }, + value: { issues: ["namespace-catalog-loading"] }, + }); + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { + providerId: "namespaces", + response: { + code: "unknown", + epoch, + retry: "never", + status: "failed", + }, + scope: "connection", + status: "usable", + }, + })).toMatchObject({ + source: { outcome: "failed" }, + value: { issues: ["namespace-catalog-failed"] }, + }); + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { + reason: "malformed-response", + status: "unavailable", + }, + })).toMatchObject({ + source: { + outcome: "unavailable", + reason: "malformed-response", + }, + value: { issues: ["namespace-catalog-malformed"] }, + }); + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { + reason: "provider-failed", + status: "unavailable", + }, + })).toMatchObject({ + value: { issues: ["namespace-catalog-failed"] }, + }); + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { status: "cancelled" }, + })).toBeNull(); + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { status: "superseded" }, + })).toBeNull(); + }); +}); diff --git a/src/vnext/namespace-catalog-boundary.ts b/src/vnext/namespace-catalog-boundary.ts new file mode 100644 index 0000000..78c8113 --- /dev/null +++ b/src/vnext/namespace-catalog-boundary.ts @@ -0,0 +1,558 @@ +import type { SqlCatalogEpoch } from "./relation-completion-types.js"; +import type { + SqlCanonicalNamespacePath, + SqlNamespaceCatalogContainer, + SqlNamespaceCatalogResolvedContainer, + SqlNamespaceCatalogSearchRequest, + SqlNamespaceCatalogSearchResponse, + SqlNamespaceContainerRole, + SqlNamespacePathComponent, +} from "./namespace-catalog-types.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, +} from "./types.js"; + +export const MAX_NAMESPACE_PROVIDER_ID_LENGTH = 256; +export const MAX_NAMESPACE_SCOPE_LENGTH = 512; +export const MAX_NAMESPACE_DIALECT_ID_LENGTH = 128; +export const MAX_NAMESPACE_EPOCH_TOKEN_LENGTH = 256; +export const MAX_NAMESPACE_ENTITY_ID_LENGTH = 256; +export const MAX_NAMESPACE_IDENTIFIER_LENGTH = 256; +export const MAX_NAMESPACE_INSERT_TEXT_LENGTH = 1_024; +export const MAX_NAMESPACE_DETAIL_LENGTH = 1_024; +export const MAX_NAMESPACE_PATH_COMPONENTS = 32; +export const MAX_NAMESPACE_SEARCH_PATHS = 32; +export const MAX_NAMESPACE_SEARCH_PATH_COMPONENTS = 8; +export const MAX_NAMESPACE_RESULTS = 128; + +const capturedProviderBrand: unique symbol = Symbol( + "CapturedSqlNamespaceCatalogProvider", +); + +export interface CapturedSqlNamespaceCatalogProvider { + readonly [capturedProviderBrand]: + "CapturedSqlNamespaceCatalogProvider"; +} + +export interface SqlCapturedNamespaceCatalogProviderContext { + readonly id: string; + readonly search: ( + this: void, + request: SqlNamespaceCatalogSearchRequest, + signal: AbortSignal, + ) => unknown; +} + +export type SqlNamespaceBoundaryFailureReason = + | "duplicate-entity-id" + | "invalid-shape" + | "resource-limit"; + +export type SqlNamespaceBoundaryResult = + | { + readonly status: "accepted"; + readonly value: Value; + } + | { + readonly reason: SqlNamespaceBoundaryFailureReason; + readonly status: "malformed"; + }; + +interface DataRecord { + readonly fields: ReadonlyMap; +} + +const capturedProviders = new WeakMap< + object, + { readonly id: string; readonly search: Function } +>(); + +function accepted( + value: Value, +): SqlNamespaceBoundaryResult { + return Object.freeze({ status: "accepted", value }); +} + +function malformed( + reason: SqlNamespaceBoundaryFailureReason, +): SqlNamespaceBoundaryResult { + return Object.freeze({ reason, status: "malformed" }); +} + +function record( + value: unknown, + allowed: ReadonlySet, +): DataRecord | null { + if (value === null || typeof value !== "object") return null; + let keys: readonly PropertyKey[]; + try { + keys = Reflect.ownKeys(value); + } catch { + return null; + } + const fields = new Map(); + for (const key of keys) { + if (typeof key !== "string" || !allowed.has(key)) return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, key); + } catch { + return null; + } + if (!descriptor || !("value" in descriptor)) return null; + fields.set(key, descriptor.value); + } + return { fields }; +} + +function required( + source: DataRecord, + key: string, +): unknown { + return source.fields.has(key) + ? source.fields.get(key) + : undefined; +} + +function boundedString( + value: unknown, + maximum: number, + allowEmpty = false, +): string | null { + return typeof value === "string" && + value.length <= maximum && + (allowEmpty || value.length > 0) + ? value + : null; +} + +function arrayLength( + value: unknown, + maximum: number, +): number | null { + if (!Array.isArray(value)) return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, "length"); + } catch { + return null; + } + const length = descriptor && "value" in descriptor + ? descriptor.value + : undefined; + return typeof length === "number" && + Number.isSafeInteger(length) && + length >= 0 && + length <= maximum + ? length + : null; +} + +function arrayElement( + value: unknown, + index: number, +): unknown | null { + if (!Array.isArray(value)) return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, String(index)); + } catch { + return null; + } + return descriptor && "value" in descriptor + ? descriptor.value + : null; +} + +function identifier( + value: unknown, +): SqlIdentifierComponent | null { + const source = record(value, new Set(["quoted", "value"])); + if (!source) return null; + const quoted = required(source, "quoted"); + const text = boundedString( + required(source, "value"), + MAX_NAMESPACE_IDENTIFIER_LENGTH, + true, + ); + return typeof quoted === "boolean" && text !== null + ? Object.freeze({ quoted, value: text }) + : null; +} + +function identifierPath( + value: unknown, + maximum: number, +): SqlIdentifierPath | null { + const length = arrayLength(value, maximum); + if (length === null) return null; + const output: SqlIdentifierComponent[] = []; + for (let index = 0; index < length; index += 1) { + const component = identifier(arrayElement(value, index)); + if (!component) return null; + output.push(component); + } + return Object.freeze(output); +} + +function searchPaths( + value: unknown, +): readonly SqlIdentifierPath[] | null { + const length = arrayLength(value, MAX_NAMESPACE_SEARCH_PATHS); + if (length === null) return null; + const output: SqlIdentifierPath[] = []; + for (let index = 0; index < length; index += 1) { + const path = identifierPath( + arrayElement(value, index), + MAX_NAMESPACE_SEARCH_PATH_COMPONENTS, + ); + if (!path) return null; + output.push(path); + } + return Object.freeze(output); +} + +function epoch(value: unknown): SqlCatalogEpoch | null { + const source = record(value, new Set(["generation", "token"])); + if (!source) return null; + const generation = required(source, "generation"); + const token = boundedString( + required(source, "token"), + MAX_NAMESPACE_EPOCH_TOKEN_LENGTH, + ); + return typeof generation === "number" && + Number.isSafeInteger(generation) && + generation >= 0 && + token !== null + ? Object.freeze({ generation, token }) + : null; +} + +function sameEpoch( + left: SqlCatalogEpoch, + right: SqlCatalogEpoch, +): boolean { + return left.generation === right.generation && + left.token === right.token; +} + +function role(value: unknown): SqlNamespaceContainerRole | null { + return value === "catalog" || + value === "schema" || + value === "project" || + value === "dataset" + ? value + : null; +} + +function namespacePath( + value: unknown, +): SqlCanonicalNamespacePath | null { + const length = arrayLength(value, MAX_NAMESPACE_PATH_COMPONENTS); + if (length === null || length === 0) return null; + const output: SqlNamespacePathComponent[] = []; + for (let index = 0; index < length; index += 1) { + const source = record( + arrayElement(value, index), + new Set(["quoted", "role", "value"]), + ); + if (!source) return null; + const componentRole = role(required(source, "role")); + const component = identifier({ + quoted: required(source, "quoted"), + value: required(source, "value"), + }); + if (!componentRole || !component) return null; + output.push(Object.freeze({ ...component, role: componentRole })); + } + const first = output[0]; + if (!first) return null; + return Object.freeze([first, ...output.slice(1)]); +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function pathKey(path: SqlCanonicalNamespacePath): string { + return path.map((component) => + `${component.role}:${component.quoted ? "q" : "u"}:${component.value}` + ).join("\u0000"); +} + +function sameContainer( + left: SqlNamespaceCatalogContainer, + right: SqlNamespaceCatalogContainer, +): boolean { + return left.containerEntityId === right.containerEntityId && + left.detail === right.detail && + left.insertText === right.insertText && + left.matchQuality === right.matchQuality && + pathKey(left.canonicalPath) === pathKey(right.canonicalPath); +} + +export function captureSqlNamespaceCatalogProvider( + value: unknown, +): SqlNamespaceBoundaryResult { + const source = record(value, new Set(["id", "search"])); + const id = source + ? boundedString( + required(source, "id"), + MAX_NAMESPACE_PROVIDER_ID_LENGTH, + ) + : null; + const search = source ? required(source, "search") : null; + if (!source || id === null || typeof search !== "function") { + return malformed("invalid-shape"); + } + const capturedValue: CapturedSqlNamespaceCatalogProvider = { + [capturedProviderBrand]: "CapturedSqlNamespaceCatalogProvider", + }; + const captured = Object.freeze(capturedValue); + capturedProviders.set(captured, { id, search }); + return accepted(captured); +} + +export function resolveSqlNamespaceCatalogProvider( + provider: CapturedSqlNamespaceCatalogProvider, +): SqlCapturedNamespaceCatalogProviderContext | null { + const captured = capturedProviders.get(provider); + return captured + ? Object.freeze({ + id: captured.id, + search: ( + request: SqlNamespaceCatalogSearchRequest, + signal: AbortSignal, + ): unknown => + Reflect.apply(captured.search, undefined, [request, signal]), + }) + : null; +} + +export function createSqlNamespaceCatalogSearchRequest( + value: unknown, +): SqlNamespaceBoundaryResult { + const source = record( + value, + new Set([ + "dialectId", + "expectedEpoch", + "limit", + "prefix", + "qualifier", + "scope", + "searchPaths", + ]), + ); + if (!source) return malformed("invalid-shape"); + const dialectId = boundedString( + required(source, "dialectId"), + MAX_NAMESPACE_DIALECT_ID_LENGTH, + ); + const scope = boundedString( + required(source, "scope"), + MAX_NAMESPACE_SCOPE_LENGTH, + ); + const expectedValue = required(source, "expectedEpoch"); + const expectedEpoch = expectedValue === null + ? null + : epoch(expectedValue); + const limit = required(source, "limit"); + const prefix = identifier(required(source, "prefix")); + const qualifier = identifierPath( + required(source, "qualifier"), + MAX_NAMESPACE_PATH_COMPONENTS, + ); + const paths = searchPaths(required(source, "searchPaths")); + if ( + dialectId === null || + scope === null || + expectedEpoch === null && expectedValue !== null || + typeof limit !== "number" || + !Number.isSafeInteger(limit) || + limit < 1 || + limit > MAX_NAMESPACE_RESULTS || + !prefix || + !qualifier || + !paths + ) { + return malformed("invalid-shape"); + } + return accepted(Object.freeze({ + dialectId, + expectedEpoch, + limit, + prefix, + qualifier, + scope, + searchPaths: paths, + })); +} + +function decodeContainer( + value: unknown, + providerId: string, + scope: string, + responseEpoch: SqlCatalogEpoch, +): SqlNamespaceCatalogResolvedContainer | null { + const source = record( + value, + new Set([ + "canonicalPath", + "containerEntityId", + "detail", + "insertText", + "matchQuality", + ]), + ); + if (!source) return null; + const canonicalPath = namespacePath( + required(source, "canonicalPath"), + ); + const containerEntityId = boundedString( + required(source, "containerEntityId"), + MAX_NAMESPACE_ENTITY_ID_LENGTH, + ); + const insertText = boundedString( + required(source, "insertText"), + MAX_NAMESPACE_INSERT_TEXT_LENGTH, + ); + const matchQuality = required(source, "matchQuality"); + const rawDetail = required(source, "detail"); + const detail = rawDetail === undefined + ? undefined + : boundedString(rawDetail, MAX_NAMESPACE_DETAIL_LENGTH, true); + if ( + !canonicalPath || + containerEntityId === null || + insertText === null || + (matchQuality !== "exact" && matchQuality !== "equivalent") || + detail === null + ) { + return null; + } + return Object.freeze({ + canonicalPath, + containerEntityId, + ...(detail === undefined ? {} : { detail }), + insertText, + matchQuality, + provenance: Object.freeze({ + containerEntityId, + epoch: responseEpoch, + providerId, + scope, + }), + }); +} + +export function decodeSqlNamespaceCatalogSearchResponse( + provider: CapturedSqlNamespaceCatalogProvider, + request: SqlNamespaceCatalogSearchRequest, + value: unknown, +): SqlNamespaceBoundaryResult { + const context = resolveSqlNamespaceCatalogProvider(provider); + if (!context) return malformed("invalid-shape"); + const source = record( + value, + new Set([ + "code", + "containers", + "coverage", + "epoch", + "retry", + "status", + ]), + ); + if (!source) return malformed("invalid-shape"); + const responseEpoch = epoch(required(source, "epoch")); + if ( + !responseEpoch || + request.expectedEpoch !== null && + !sameEpoch(request.expectedEpoch, responseEpoch) + ) { + return malformed("invalid-shape"); + } + const status = required(source, "status"); + if (status === "loading") { + if (source.fields.size !== 2) return malformed("invalid-shape"); + return accepted(Object.freeze({ + epoch: responseEpoch, + status, + })); + } + if (status === "failed") { + const code = required(source, "code"); + const retry = required(source, "retry"); + if ( + source.fields.size !== 4 || + ( + code !== "authentication" && + code !== "authorization" && + code !== "invalid-configuration" && + code !== "rate-limited" && + code !== "unavailable" && + code !== "unknown" + ) || + ( + retry !== "after-invalidation" && + retry !== "never" && + retry !== "next-request" + ) + ) { + return malformed("invalid-shape"); + } + return accepted(Object.freeze({ + code, + epoch: responseEpoch, + retry, + status, + })); + } + const coverage = required(source, "coverage"); + const rawContainers = required(source, "containers"); + const length = arrayLength(rawContainers, request.limit); + if ( + status !== "ready" || + source.fields.size !== 4 || + (coverage !== "complete" && coverage !== "partial") || + length === null + ) { + return malformed("invalid-shape"); + } + const byId = new Map< + string, + SqlNamespaceCatalogResolvedContainer + >(); + for (let index = 0; index < length; index += 1) { + const container = decodeContainer( + arrayElement(rawContainers, index), + context.id, + request.scope, + responseEpoch, + ); + if (!container) return malformed("invalid-shape"); + const prior = byId.get(container.containerEntityId); + if (prior && !sameContainer(prior, container)) { + return malformed("duplicate-entity-id"); + } + byId.set(container.containerEntityId, container); + } + const containers = [...byId.values()].sort((left, right) => + (left.matchQuality === right.matchQuality + ? 0 + : left.matchQuality === "exact" + ? -1 + : 1) || + left.canonicalPath.length - right.canonicalPath.length || + compareText(pathKey(left.canonicalPath), pathKey(right.canonicalPath)) || + compareText(left.containerEntityId, right.containerEntityId) + ); + return accepted(Object.freeze({ + containers: Object.freeze(containers), + coverage, + epoch: responseEpoch, + status, + })); +} diff --git a/src/vnext/namespace-catalog-coordinator.ts b/src/vnext/namespace-catalog-coordinator.ts new file mode 100644 index 0000000..d43ba13 --- /dev/null +++ b/src/vnext/namespace-catalog-coordinator.ts @@ -0,0 +1,488 @@ +import { + captureSqlNamespaceCatalogProvider, + createSqlNamespaceCatalogSearchRequest, + decodeSqlNamespaceCatalogSearchResponse, + MAX_NAMESPACE_DIALECT_ID_LENGTH, + MAX_NAMESPACE_SCOPE_LENGTH, + resolveSqlNamespaceCatalogProvider, + type CapturedSqlNamespaceCatalogProvider, + type SqlCapturedNamespaceCatalogProviderContext, +} from "./namespace-catalog-boundary.js"; +import type { + SqlNamespaceCatalogSearchRequest, + SqlNamespaceCatalogSearchResponse, +} from "./namespace-catalog-types.js"; +import type { SqlCatalogEpoch } from "./relation-completion-types.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, +} from "./types.js"; + +export const DEFAULT_NAMESPACE_CACHE_ENTRIES = 256; +export const MAX_NAMESPACE_CACHE_ENTRIES = 4_096; + +export interface SqlNamespaceCatalogOwnerOptions { + readonly dialectId: string; + readonly scope: string; +} + +export interface SqlNamespaceCatalogSearchInput { + readonly expectedEpoch: SqlCatalogEpoch | null; + readonly limit: number; + readonly prefix: SqlIdentifierComponent; + readonly qualifier: SqlIdentifierPath; + readonly searchPaths: readonly SqlIdentifierPath[]; +} + +export type SqlNamespaceCatalogSearchOutcome = + | { + readonly providerId: string; + readonly response: SqlNamespaceCatalogSearchResponse; + readonly scope: string; + readonly status: "usable"; + } + | { + readonly status: "cancelled"; + } + | { + readonly status: "superseded"; + } + | { + readonly reason: + | "disposed" + | "invalid-request" + | "malformed-response" + | "provider-failed"; + readonly status: "unavailable"; + }; + +export interface SqlNamespaceCatalogSearchTicket { + readonly cancel: (this: void) => void; + readonly result: Promise; +} + +export interface SqlNamespaceCatalogOwner { + readonly dispose: (this: void) => void; + readonly request: ( + this: void, + input: SqlNamespaceCatalogSearchInput, + ) => SqlNamespaceCatalogSearchTicket; +} + +export type SqlNamespaceCatalogOwnerResult = + | { + readonly owner: SqlNamespaceCatalogOwner; + readonly status: "prepared"; + } + | { + readonly reason: "disposed" | "invalid-options"; + readonly status: "unavailable"; + }; + +export interface SqlNamespaceCatalogCoordinator { + readonly dispose: (this: void) => void; + readonly prepareOwner: ( + this: void, + options: SqlNamespaceCatalogOwnerOptions, + ) => SqlNamespaceCatalogOwnerResult; + readonly providerId: string; +} + +export type SqlNamespaceCatalogCoordinatorResult = + | { + readonly coordinator: SqlNamespaceCatalogCoordinator; + readonly status: "created"; + } + | { + readonly reason: "invalid-options" | "invalid-provider"; + readonly status: "unavailable"; + }; + +interface CoordinatorState { + readonly cache: Map; + readonly capturedProvider: CapturedSqlNamespaceCatalogProvider; + readonly context: SqlCapturedNamespaceCatalogProviderContext; + disposed: boolean; + readonly maxCacheEntries: number; + readonly owners: Set; +} + +interface OwnerState { + active: ConsumerState | null; + readonly dialectId: string; + disposed: boolean; + observedEpoch: SqlCatalogEpoch | null; + owner: CoordinatorState | null; + readonly scope: string; +} + +interface ConsumerState { + readonly controller: AbortController; + readonly owner: OwnerState; + resolve: + | ((value: SqlNamespaceCatalogSearchOutcome) => void) + | null; + settled: boolean; +} + +const CANCELLED: SqlNamespaceCatalogSearchOutcome = + Object.freeze({ status: "cancelled" }); +const SUPERSEDED: SqlNamespaceCatalogSearchOutcome = + Object.freeze({ status: "superseded" }); + +function unavailable( + reason: Extract< + SqlNamespaceCatalogSearchOutcome, + { readonly status: "unavailable" } + >["reason"], +): SqlNamespaceCatalogSearchOutcome { + return Object.freeze({ reason, status: "unavailable" }); +} + +function property( + value: unknown, + key: string, +): { readonly value: unknown } | null { + if (value === null || typeof value !== "object") return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, key); + } catch { + return null; + } + return descriptor && "value" in descriptor + ? { value: descriptor.value } + : null; +} + +function boundedString(value: unknown, maximum: number): string | null { + return typeof value === "string" && + value.length > 0 && + value.length <= maximum + ? value + : null; +} + +function segment(value: string): string { + return `${value.length}:${value}`; +} + +function componentKey(component: SqlIdentifierComponent): string { + return segment( + `${component.quoted ? "q" : "u"}${component.value}`, + ); +} + +function pathKey(path: SqlIdentifierPath): string { + return segment(path.map(componentKey).join("")); +} + +function cacheKey( + request: SqlNamespaceCatalogSearchRequest, + responseEpoch: SqlCatalogEpoch, +): string { + return [ + segment(request.scope), + segment(request.dialectId), + segment(String(responseEpoch.generation)), + segment(responseEpoch.token), + pathKey(request.qualifier), + componentKey(request.prefix), + segment(request.searchPaths.map(pathKey).join("")), + segment(String(request.limit)), + ].join(""); +} + +function cacheGet( + state: CoordinatorState, + key: string, +): SqlNamespaceCatalogSearchResponse | null { + const value = state.cache.get(key); + if (!value) return null; + state.cache.delete(key); + state.cache.set(key, value); + return value; +} + +function cacheSet( + state: CoordinatorState, + key: string, + value: SqlNamespaceCatalogSearchResponse, +): void { + state.cache.delete(key); + state.cache.set(key, value); + while (state.cache.size > state.maxCacheEntries) { + const oldest = state.cache.keys().next().value; + if (typeof oldest !== "string") break; + state.cache.delete(oldest); + } +} + +function settle( + consumer: ConsumerState, + outcome: SqlNamespaceCatalogSearchOutcome, +): void { + if (consumer.settled) return; + consumer.settled = true; + if (consumer.owner.active === consumer) { + consumer.owner.active = null; + } + const resolve = consumer.resolve; + consumer.resolve = null; + resolve?.(outcome); +} + +function cancel( + consumer: ConsumerState, + outcome: SqlNamespaceCatalogSearchOutcome, +): void { + if (consumer.settled) return; + consumer.controller.abort(); + settle(consumer, outcome); +} + +function settledTicket( + outcome: SqlNamespaceCatalogSearchOutcome, +): SqlNamespaceCatalogSearchTicket { + return Object.freeze({ + cancel: (): void => {}, + result: Promise.resolve(outcome), + }); +} + +function providerWork( + state: CoordinatorState, + owner: OwnerState, + request: SqlNamespaceCatalogSearchRequest, + consumer: ConsumerState, +): void { + let pending: unknown; + try { + pending = state.context.search( + request, + consumer.controller.signal, + ); + } catch { + settle(consumer, unavailable("provider-failed")); + return; + } + Promise.resolve(pending).then( + (value) => { + if ( + consumer.settled || + consumer.controller.signal.aborted || + state.disposed || + owner.disposed || + owner.owner !== state + ) { + return; + } + const decoded = decodeSqlNamespaceCatalogSearchResponse( + state.capturedProvider, + request, + value, + ); + if (decoded.status === "malformed") { + settle(consumer, unavailable("malformed-response")); + return; + } + owner.observedEpoch = decoded.value.epoch; + if ( + decoded.value.status === "ready" && + decoded.value.coverage === "complete" + ) { + cacheSet( + state, + cacheKey(request, decoded.value.epoch), + decoded.value, + ); + } + settle(consumer, Object.freeze({ + providerId: state.context.id, + response: decoded.value, + scope: owner.scope, + status: "usable", + })); + }, + () => { + if (!consumer.settled) { + settle(consumer, unavailable("provider-failed")); + } + }, + ); +} + +function request( + owner: OwnerState, + input: unknown, +): SqlNamespaceCatalogSearchTicket { + const state = owner.owner; + if (!state || state.disposed || owner.disposed) { + return settledTicket(unavailable("disposed")); + } + const expected = property(input, "expectedEpoch"); + const limit = property(input, "limit"); + const prefix = property(input, "prefix"); + const qualifier = property(input, "qualifier"); + const paths = property(input, "searchPaths"); + if (!expected || !limit || !prefix || !qualifier || !paths) { + return settledTicket(unavailable("invalid-request")); + } + const created = createSqlNamespaceCatalogSearchRequest({ + dialectId: owner.dialectId, + expectedEpoch: expected.value === null + ? owner.observedEpoch + : expected.value, + limit: limit.value, + prefix: prefix.value, + qualifier: qualifier.value, + scope: owner.scope, + searchPaths: paths.value, + }); + if (created.status === "malformed") { + return settledTicket(unavailable("invalid-request")); + } + if (owner.active) cancel(owner.active, SUPERSEDED); + const normalized = created.value; + const key = normalized.expectedEpoch === null + ? null + : cacheKey(normalized, normalized.expectedEpoch); + const cached = key === null ? null : cacheGet(state, key); + if (cached) { + return settledTicket(Object.freeze({ + providerId: state.context.id, + response: cached, + scope: owner.scope, + status: "usable", + })); + } + let resolveResult: + (value: SqlNamespaceCatalogSearchOutcome) => void = + (): void => {}; + const result = new Promise( + (resolve) => { + resolveResult = resolve; + }, + ); + const consumer: ConsumerState = { + controller: new AbortController(), + owner, + resolve: resolveResult, + settled: false, + }; + owner.active = consumer; + const ticket = Object.freeze({ + cancel: (): void => cancel(consumer, CANCELLED), + result, + }); + providerWork(state, owner, normalized, consumer); + return ticket; +} + +function disposeOwner(owner: OwnerState): void { + if (owner.disposed) return; + owner.disposed = true; + if (owner.active) cancel(owner.active, unavailable("disposed")); + owner.owner?.owners.delete(owner); + owner.owner = null; +} + +function prepareOwner( + state: CoordinatorState, + value: unknown, +): SqlNamespaceCatalogOwnerResult { + if (state.disposed) { + return Object.freeze({ reason: "disposed", status: "unavailable" }); + } + const dialectId = boundedString( + property(value, "dialectId")?.value, + MAX_NAMESPACE_DIALECT_ID_LENGTH, + ); + const scope = boundedString( + property(value, "scope")?.value, + MAX_NAMESPACE_SCOPE_LENGTH, + ); + if (!dialectId || !scope) { + return Object.freeze({ + reason: "invalid-options", + status: "unavailable", + }); + } + const owner: OwnerState = { + active: null, + dialectId, + disposed: false, + observedEpoch: null, + owner: state, + scope, + }; + state.owners.add(owner); + return Object.freeze({ + owner: Object.freeze({ + dispose: (): void => disposeOwner(owner), + request: (input: SqlNamespaceCatalogSearchInput) => + request(owner, input), + }), + status: "prepared", + }); +} + +function dispose(state: CoordinatorState): void { + if (state.disposed) return; + state.disposed = true; + state.cache.clear(); + for (const owner of state.owners) disposeOwner(owner); +} + +export function createSqlNamespaceCatalogCoordinator( + value: unknown, +): SqlNamespaceCatalogCoordinatorResult { + const captured = captureSqlNamespaceCatalogProvider( + property(value, "provider")?.value, + ); + if (captured.status === "malformed") { + return Object.freeze({ + reason: "invalid-provider", + status: "unavailable", + }); + } + const context = resolveSqlNamespaceCatalogProvider(captured.value); + if (!context) { + return Object.freeze({ + reason: "invalid-provider", + status: "unavailable", + }); + } + const rawMaximum = property(value, "maxCacheEntries"); + const maximum = rawMaximum === null + ? DEFAULT_NAMESPACE_CACHE_ENTRIES + : rawMaximum.value; + if ( + typeof maximum !== "number" || + !Number.isSafeInteger(maximum) || + maximum < 1 || + maximum > MAX_NAMESPACE_CACHE_ENTRIES + ) { + return Object.freeze({ + reason: "invalid-options", + status: "unavailable", + }); + } + const state: CoordinatorState = { + cache: new Map(), + capturedProvider: captured.value, + context, + disposed: false, + maxCacheEntries: maximum, + owners: new Set(), + }; + return Object.freeze({ + coordinator: Object.freeze({ + dispose: (): void => dispose(state), + prepareOwner: (options: SqlNamespaceCatalogOwnerOptions) => + prepareOwner(state, options), + providerId: context.id, + }), + status: "created", + }); +} diff --git a/src/vnext/namespace-catalog-types.ts b/src/vnext/namespace-catalog-types.ts new file mode 100644 index 0000000..211a742 --- /dev/null +++ b/src/vnext/namespace-catalog-types.ts @@ -0,0 +1,96 @@ +import type { SqlCatalogEpoch } from "./relation-completion-types.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, + SqlTextRange, +} from "./types.js"; + +export type SqlNamespaceContainerRole = + | "catalog" + | "schema" + | "project" + | "dataset"; + +export interface SqlNamespacePathComponent + extends SqlIdentifierComponent { + readonly role: SqlNamespaceContainerRole; +} + +export type SqlCanonicalNamespacePath = + readonly [ + SqlNamespacePathComponent, + ...SqlNamespacePathComponent[], + ]; + +export interface SqlNamespaceCatalogSearchRequest { + readonly dialectId: string; + readonly expectedEpoch: SqlCatalogEpoch | null; + readonly limit: number; + readonly prefix: SqlIdentifierComponent; + readonly qualifier: SqlIdentifierPath; + readonly scope: string; + readonly searchPaths: readonly SqlIdentifierPath[]; +} + +export interface SqlNamespaceCatalogContainer { + readonly canonicalPath: SqlCanonicalNamespacePath; + readonly containerEntityId: string; + readonly detail?: string; + readonly insertText: string; + readonly matchQuality: "equivalent" | "exact"; +} + +export type SqlNamespaceCatalogSearchResponse = + | { + readonly containers: + readonly SqlNamespaceCatalogResolvedContainer[]; + readonly coverage: "complete" | "partial"; + readonly epoch: SqlCatalogEpoch; + readonly status: "ready"; + } + | { + readonly epoch: SqlCatalogEpoch; + readonly status: "loading"; + } + | { + readonly code: + | "authentication" + | "authorization" + | "invalid-configuration" + | "rate-limited" + | "unavailable" + | "unknown"; + readonly epoch: SqlCatalogEpoch; + readonly retry: + | "after-invalidation" + | "never" + | "next-request"; + readonly status: "failed"; + }; + +export interface SqlNamespaceCatalogProvenance { + readonly containerEntityId: string; + readonly epoch: SqlCatalogEpoch; + readonly providerId: string; + readonly scope: string; +} + +export interface SqlNamespaceCatalogResolvedContainer + extends SqlNamespaceCatalogContainer { + readonly provenance: SqlNamespaceCatalogProvenance; +} + +export interface SqlNamespaceCatalogProvider { + readonly id: string; + readonly search: ( + this: void, + request: SqlNamespaceCatalogSearchRequest, + signal: AbortSignal, + ) => Promise; +} + +export interface SqlNamespaceQuerySite { + readonly prefix: SqlIdentifierComponent; + readonly qualifier: SqlIdentifierPath; + readonly replacementRange: SqlTextRange; +} diff --git a/src/vnext/namespace-completion.ts b/src/vnext/namespace-completion.ts new file mode 100644 index 0000000..c4cb1bd --- /dev/null +++ b/src/vnext/namespace-completion.ts @@ -0,0 +1,293 @@ +import type { + SqlNamespaceCatalogSearchInput, + SqlNamespaceCatalogSearchOutcome, +} from "./namespace-catalog-coordinator.js"; +import type { + SqlNamespaceCatalogResolvedContainer, + SqlNamespaceContainerRole, + SqlNamespaceQuerySite, +} from "./namespace-catalog-types.js"; +import type { + SqlCatalogEpoch, +} from "./relation-completion-types.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, + SqlTextRange, +} from "./types.js"; + +export interface SqlNamespaceCompletionProvenance { + readonly containerEntityId: string; + readonly epoch: SqlCatalogEpoch; + readonly kind: "namespace-catalog"; + readonly providerId: string; + readonly scope: string; +} + +export interface SqlNamespaceCompletionItem { + readonly detail?: string; + readonly edit: { + readonly from: number; + readonly insert: string; + readonly to: number; + }; + readonly label: string; + readonly role: SqlNamespaceContainerRole; + readonly provenance: SqlNamespaceCompletionProvenance; +} + +export type SqlNamespaceCompletionIssue = + | "namespace-catalog-failed" + | "namespace-catalog-loading" + | "namespace-catalog-malformed" + | "namespace-catalog-partial" + | "namespace-prefix-uncertain" + | "result-limit"; + +export interface SqlNamespaceCompletionList { + readonly isIncomplete: boolean; + readonly issues: readonly SqlNamespaceCompletionIssue[]; + readonly items: readonly SqlNamespaceCompletionItem[]; +} + +export type SqlNamespaceCatalogProviderReport = + | { + readonly coverage: "complete" | "partial"; + readonly outcome: "ready"; + readonly providerId: string; + } + | { + readonly outcome: "loading"; + readonly providerId: string; + } + | { + readonly outcome: "failed"; + readonly providerId: string; + } + | { + readonly outcome: "unavailable"; + readonly providerId: string; + readonly reason: + | "disposed" + | "invalid-request" + | "malformed-response" + | "provider-failed"; + }; + +export interface SqlNamespaceCompletionComposition { + readonly source: SqlNamespaceCatalogProviderReport; + readonly value: SqlNamespaceCompletionList; +} + +export type SqlNamespacePrefixMatcher = ( + this: void, + candidate: SqlIdentifierComponent, + prefix: SqlIdentifierComponent, +) => "match" | "no-match" | "unknown"; + +const ROLE_ORDER: Readonly> = + Object.freeze({ + catalog: 0, + project: 1, + schema: 2, + dataset: 3, + }); + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function last(values: readonly Value[]): Value | null { + return values[values.length - 1] ?? null; +} + +function pathText( + container: SqlNamespaceCatalogResolvedContainer, +): string { + return container.canonicalPath.map((component) => + `${component.role}:${component.quoted ? "q" : "u"}:${component.value}` + ).join("\u0000"); +} + +function compareContainers( + left: SqlNamespaceCatalogResolvedContainer, + right: SqlNamespaceCatalogResolvedContainer, +): number { + const leftLast = last(left.canonicalPath); + const rightLast = last(right.canonicalPath); + return ( + (left.matchQuality === right.matchQuality + ? 0 + : left.matchQuality === "exact" + ? -1 + : 1) || + left.canonicalPath.length - right.canonicalPath.length || + ( + leftLast && rightLast + ? ROLE_ORDER[leftLast.role] - ROLE_ORDER[rightLast.role] + : 0 + ) || + compareText(leftLast?.value ?? "", rightLast?.value ?? "") || + compareText(pathText(left), pathText(right)) || + compareText( + left.provenance.containerEntityId, + right.provenance.containerEntityId, + ) + ); +} + +function list( + items: readonly SqlNamespaceCompletionItem[], + issues: readonly SqlNamespaceCompletionIssue[], +): SqlNamespaceCompletionList { + return Object.freeze({ + isIncomplete: issues.length > 0, + issues: Object.freeze(issues), + items: Object.freeze(items), + }); +} + +export function prepareSqlNamespaceCatalogSearch( + site: SqlNamespaceQuerySite, + expectedEpoch: SqlCatalogEpoch | null, + searchPaths: readonly SqlIdentifierPath[], + limit: number, +): SqlNamespaceCatalogSearchInput { + return Object.freeze({ + expectedEpoch, + limit, + prefix: site.prefix, + qualifier: site.qualifier, + searchPaths, + }); +} + +function unavailable( + providerId: string, + reason: Extract< + SqlNamespaceCatalogSearchOutcome, + { readonly status: "unavailable" } + >["reason"], +): SqlNamespaceCompletionComposition { + return Object.freeze({ + source: Object.freeze({ + outcome: "unavailable", + providerId, + reason, + }), + value: list( + [], + [reason === "malformed-response" + ? "namespace-catalog-malformed" + : "namespace-catalog-failed"], + ), + }); +} + +function item( + container: SqlNamespaceCatalogResolvedContainer, + replacementRange: SqlTextRange, +): SqlNamespaceCompletionItem | null { + const component = last(container.canonicalPath); + if (!component) return null; + return Object.freeze({ + ...(container.detail === undefined + ? {} + : { detail: container.detail }), + edit: Object.freeze({ + from: replacementRange.from, + insert: container.insertText, + to: replacementRange.to, + }), + label: component.value, + provenance: Object.freeze({ + containerEntityId: container.provenance.containerEntityId, + epoch: container.provenance.epoch, + kind: "namespace-catalog", + providerId: container.provenance.providerId, + scope: container.provenance.scope, + }), + role: component.role, + }); +} + +export function composeSqlNamespaceCompletion( + input: { + readonly matchPrefix: SqlNamespacePrefixMatcher; + readonly outcome: SqlNamespaceCatalogSearchOutcome; + readonly prefix: SqlIdentifierComponent; + readonly providerId: string; + readonly replacementRange: SqlTextRange; + }, +): SqlNamespaceCompletionComposition | null { + if ( + input.outcome.status === "cancelled" || + input.outcome.status === "superseded" + ) { + return null; + } + if (input.outcome.status === "unavailable") { + return unavailable(input.providerId, input.outcome.reason); + } + const response = input.outcome.response; + if (response.status === "loading") { + return Object.freeze({ + source: Object.freeze({ + outcome: "loading", + providerId: input.outcome.providerId, + }), + value: list([], ["namespace-catalog-loading"]), + }); + } + if (response.status === "failed") { + return Object.freeze({ + source: Object.freeze({ + outcome: "failed", + providerId: input.outcome.providerId, + }), + value: list([], ["namespace-catalog-failed"]), + }); + } + const issues = new Set(); + if (response.coverage === "partial") { + issues.add("namespace-catalog-partial"); + } + const seen = new Set(); + const containers: SqlNamespaceCatalogResolvedContainer[] = []; + for (const container of response.containers) { + const component = last(container.canonicalPath); + if (!component) continue; + let match: ReturnType; + try { + match = input.matchPrefix(component, input.prefix); + } catch { + match = "unknown"; + } + if (match === "unknown") { + issues.add("namespace-prefix-uncertain"); + continue; + } + if (match === "no-match") continue; + const identity = [ + container.provenance.providerId, + container.provenance.scope, + container.provenance.containerEntityId, + ].join("\u0000"); + if (seen.has(identity)) continue; + seen.add(identity); + containers.push(container); + } + containers.sort(compareContainers); + const items = containers.flatMap((container) => { + const value = item(container, input.replacementRange); + return value ? [value] : []; + }); + return Object.freeze({ + source: Object.freeze({ + coverage: response.coverage, + outcome: "ready", + providerId: input.outcome.providerId, + }), + value: list(items, [...issues]), + }); +} From e59423c0871cd662c0dadb46ed544ca1fa555355 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 02:59:47 +0800 Subject: [PATCH 11/25] feat(vnext): integrate namespace completion --- docs/vnext/marimo-sql-migration.md | 25 +- docs/vnext/namespace-catalog-authority.md | 9 +- src/vnext/__tests__/session.test.ts | 144 ++++++++++ .../sql-editor-completion-gate.test.ts | 57 ++++ src/vnext/codemirror/sql-editor.ts | 1 + src/vnext/index.ts | 14 + src/vnext/namespace-completion.ts | 38 +-- src/vnext/relation-completion-types.ts | 49 +++- src/vnext/session.ts | 266 +++++++++++++++++- src/vnext/types.ts | 4 + .../marimo-sql-migration.test-d.ts | 40 +++ 11 files changed, 591 insertions(+), 56 deletions(-) diff --git a/docs/vnext/marimo-sql-migration.md b/docs/vnext/marimo-sql-migration.md index fc845d9..48b9aae 100644 --- a/docs/vnext/marimo-sql-migration.md +++ b/docs/vnext/marimo-sql-migration.md @@ -1,6 +1,6 @@ # Marimo SQL Completion Migration -Status: implementation fixture; relation and column completion +Status: implementation fixture; relation, column, and namespace completion The compile-only fixture [`marimo-sql-migration.test-d.ts`](../../test/vnext-types/marimo-sql-migration.test-d.ts) @@ -121,20 +121,17 @@ explicit, tested generic-dialect policy. Marimo's current `tablesCompletionSource()` is broader than its name. The CodeMirror SQL schema source provides relations, namespace navigation, and -columns. vNext now provides the relation and lazy batched column portions, +columns. vNext now provides all three through separate bounded providers, including qualified and unqualified query-site completion, ambiguity handling, -stable provenance, cancellation, and bounded provider work. +stable provenance, cancellation, and batched column work. The remaining feature gaps are: -- namespace/container completion for database, schema, project, and dataset - navigation; and - the dialect coverage described above. -The fixture defines marimo's immutable namespace projection—stable entity ID, -scope, canonical identifier path, and namespace kind—but deliberately does -not invent a provider import. Once a public namespace provider lands, that -projection should feed one scoped provider on the shared service. +The fixture feeds marimo's immutable namespace projection—stable entity ID, +scope, canonical identifier path, and namespace kind—through one public, +scoped namespace provider on the shared service. ## Migration sequence @@ -148,11 +145,9 @@ projection should feed one scoped provider on the shared service. 4. Compare relation and column results with the golden corpus, including quoted insert text, aliases, ambiguity, partial/loading/failure states, and cold epoch behavior. -5. Add the public namespace provider and connect the prepared marimo namespace - projection. -6. Cut over the four supported dialects as one source replacement, preserving +5. Cut over the four supported dialects as one source replacement, preserving variable and keyword external sources. -7. Add dialect coverage, expand the router, and remove completion-only legacy +6. Add dialect coverage, expand the router, and remove completion-only legacy schema code. Keep legacy schema data while hover or diagnostics still use it. @@ -160,8 +155,8 @@ projection should feed one scoped provider on the shared service. The compile-only marimo fixture proves: -- one shared service configured with one relation provider and one batched - column provider; +- one shared service configured with one relation provider, one batched + column provider, and one namespace provider; - a two-relation cold column request with `expectedEpoch: null`; - stable relation and column IDs, canonical identifiers, and distinct provider-rendered insert text; diff --git a/docs/vnext/namespace-catalog-authority.md b/docs/vnext/namespace-catalog-authority.md index e3ef97d..b71c098 100644 --- a/docs/vnext/namespace-catalog-authority.md +++ b/docs/vnext/namespace-catalog-authority.md @@ -1,6 +1,6 @@ # vNext Namespace Catalog Authority -Status: integration-ready internal vertical slice +Status: public provider and session integration Namespace completion searches catalog, schema, project, and dataset containers. One query site produces one bounded provider request containing its qualifier, @@ -30,8 +30,9 @@ A newer request supersedes and aborts prior work for the same owner. Explicit cancellation and owner/coordinator disposal abort pending work. Provider throws, rejections, malformed data, and late settlements are contained. -Session integration should prepare one owner for each live scope/dialect and -dispose it when catalog authority changes. Query-site integration calls +Session integration prepares one owner for each live scope/dialect and +disposes it when catalog authority changes. Query-site integration calls `prepareSqlNamespaceCatalogSearch`, submits the result through the owner, and passes the outcome plus the site's replacement range and dialect prefix matcher -to `composeSqlNamespaceCompletion`. +to `composeSqlNamespaceCompletion`. Namespace items are merged with local and +relation-catalog items under the same bounded completion response budget. diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index eb64669..0507d4a 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -7,6 +7,7 @@ import { postgresDialect, SqlSessionError, type SqlColumnCatalogProvider, + type SqlNamespaceCatalogProvider, } from "../index.js"; import { DefaultSqlLanguageService, @@ -1428,6 +1429,149 @@ describe("column completion session integration", () => { }); }); +describe("namespace completion session integration", () => { + const relationCatalog: SqlRelationCatalogProvider = { + id: "relations", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + status: "ready", + }), + }; + + function serviceWithNamespaces( + search: SqlNamespaceCatalogProvider["search"], + budget = 40, + ) { + return createSqlLanguageService({ + catalog: relationCatalog, + completion: { catalogResponseBudgetMs: budget }, + dialects: [duckdb], + namespaces: { id: "namespaces", search }, + }); + } + + it("merges namespace containers into relation-site completion", async () => { + const requests: Parameters< + SqlNamespaceCatalogProvider["search"] + >[0][] = []; + const service = serviceWithNamespaces(async (request) => { + requests.push(request); + return { + containers: [{ + canonicalPath: [{ + quoted: false, + role: "schema", + value: "main", + }], + containerEntityId: "schema:main", + detail: "DuckDB schema", + insertText: "main", + matchQuality: "exact", + }], + coverage: "complete", + epoch: { generation: 1, token: "epoch-1" }, + status: "ready", + }; + }); + const text = "SELECT * FROM ma"; + const session = service.openDocument({ + context: { + catalog: { + scope: "connection:1", + searchPath: [[{ quoted: false, value: "main" }]], + }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + + await expect(session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + sources: [ + { feature: "relation-catalog" }, + { + feature: "namespace-catalog", + outcome: "ready", + providerId: "namespaces", + }, + ], + status: "ready", + value: { + isIncomplete: false, + items: [{ + detail: "DuckDB schema", + edit: { from: 14, insert: "main", to: 16 }, + kind: "namespace", + label: "main", + provenance: { + containerEntityId: "schema:main", + kind: "namespace-catalog", + providerId: "namespaces", + scope: "connection:1", + }, + role: "schema", + }], + }, + }); + expect(requests).toHaveLength(1); + expect(requests[0]).toMatchObject({ + dialectId: "duckdb", + expectedEpoch: null, + limit: 128, + prefix: { quoted: false, value: "ma" }, + qualifier: [], + scope: "connection:1", + }); + service.dispose(); + }); + + it("keeps slow namespace providers inside the shared budget", async () => { + vi.useFakeTimers(); + try { + const service = serviceWithNamespaces( + () => new Promise(() => {}), + 0, + ); + const text = "SELECT * FROM "; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const completion = session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }); + await vi.advanceTimersByTimeAsync(0); + await expect(completion).resolves.toMatchObject({ + sources: [ + { feature: "relation-catalog" }, + { + feature: "namespace-catalog", + outcome: "loading", + }, + ], + status: "ready", + value: { + isIncomplete: true, + issues: [{ reason: "namespace-catalog-loading" }], + }, + }); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); +}); + describe("statement-index session cache", () => { it("binds lexical behavior through authentic dialect handles", () => { const service = new DefaultSqlLanguageService({ diff --git a/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts index 6e79d23..dc306e5 100644 --- a/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts +++ b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts @@ -218,3 +218,60 @@ test("vNext editor applies a batched column completion", async () => { service.dispose(); parent.remove(); }); + +test("vNext editor exposes namespace containers at relation sites", async () => { + const parent = document.createElement("div"); + document.body.append(parent); + let namespaceCalls = 0; + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + namespaces: { + id: "browser-namespaces", + search: async () => { + namespaceCalls += 1; + return { + containers: [{ + canonicalPath: [{ + quoted: false, + role: "schema", + value: "main", + }], + containerEntityId: "schema:main", + insertText: "main", + matchQuality: "exact", + }], + coverage: "complete", + epoch: { generation: 1, token: "epoch-1" }, + status: "ready", + }; + }, + }, + }); + const support = sqlEditor({ + initialContext: { + catalog: { scope: "browser-namespaces" }, + dialect: "duckdb", + }, + service, + }); + const documentText = "SELECT * FROM ma"; + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + selection: { anchor: documentText.length }, + }); + + expect(startCompletion(view)).toBe(true); + await expect.poll(() => + currentCompletions(view.state).map((item) => ({ + label: item.label, + type: item.type, + })) + ).toEqual([{ label: "main", type: "namespace" }]); + expect(namespaceCalls).toBe(1); + + view.destroy(); + service.dispose(); + parent.remove(); +}); diff --git a/src/vnext/codemirror/sql-editor.ts b/src/vnext/codemirror/sql-editor.ts index 887b458..24f1cba 100644 --- a/src/vnext/codemirror/sql-editor.ts +++ b/src/vnext/codemirror/sql-editor.ts @@ -204,6 +204,7 @@ function haveOneEditRange(items: readonly SqlCompletionItem[]): boolean { function completionType(item: SqlCompletionItem): string { if (item.kind === "column") return "property"; + if (item.kind === "namespace") return "namespace"; return item.relationKind === "cte" ? "type" : "table"; } diff --git a/src/vnext/index.ts b/src/vnext/index.ts index 6753adb..91de922 100644 --- a/src/vnext/index.ts +++ b/src/vnext/index.ts @@ -16,6 +16,18 @@ export type { SqlColumnCatalogRelationResult, SqlColumnCatalogResolvedColumn, } from "./column-catalog-types.js"; +export type { + SqlCanonicalNamespacePath, + SqlNamespaceCatalogContainer, + SqlNamespaceCatalogProvider, + SqlNamespaceCatalogProvenance, + SqlNamespaceCatalogResolvedContainer, + SqlNamespaceCatalogSearchRequest, + SqlNamespaceCatalogSearchResponse, + SqlNamespaceContainerRole, + SqlNamespacePathComponent, + SqlNamespaceQuerySite, +} from "./namespace-catalog-types.js"; export type { OpenSqlDocument, SqlCatalogContext, @@ -70,6 +82,8 @@ export type { SqlCompletionIssue, SqlCompletionRequest, SqlCompletionRefreshToken, + SqlNamespaceCatalogProviderReport, + SqlNamespaceCompletionProvenance, SqlCompletionTrigger, SqlDisposable, SqlRelationCatalogProvider, diff --git a/src/vnext/namespace-completion.ts b/src/vnext/namespace-completion.ts index c4cb1bd..0c70406 100644 --- a/src/vnext/namespace-completion.ts +++ b/src/vnext/namespace-completion.ts @@ -9,6 +9,8 @@ import type { } from "./namespace-catalog-types.js"; import type { SqlCatalogEpoch, + SqlNamespaceCatalogProviderReport, + SqlNamespaceCompletionProvenance, } from "./relation-completion-types.js"; import type { SqlIdentifierComponent, @@ -16,14 +18,6 @@ import type { SqlTextRange, } from "./types.js"; -export interface SqlNamespaceCompletionProvenance { - readonly containerEntityId: string; - readonly epoch: SqlCatalogEpoch; - readonly kind: "namespace-catalog"; - readonly providerId: string; - readonly scope: string; -} - export interface SqlNamespaceCompletionItem { readonly detail?: string; readonly edit: { @@ -50,30 +44,6 @@ export interface SqlNamespaceCompletionList { readonly items: readonly SqlNamespaceCompletionItem[]; } -export type SqlNamespaceCatalogProviderReport = - | { - readonly coverage: "complete" | "partial"; - readonly outcome: "ready"; - readonly providerId: string; - } - | { - readonly outcome: "loading"; - readonly providerId: string; - } - | { - readonly outcome: "failed"; - readonly providerId: string; - } - | { - readonly outcome: "unavailable"; - readonly providerId: string; - readonly reason: - | "disposed" - | "invalid-request" - | "malformed-response" - | "provider-failed"; - }; - export interface SqlNamespaceCompletionComposition { readonly source: SqlNamespaceCatalogProviderReport; readonly value: SqlNamespaceCompletionList; @@ -171,6 +141,7 @@ function unavailable( ): SqlNamespaceCompletionComposition { return Object.freeze({ source: Object.freeze({ + feature: "namespace-catalog", outcome: "unavailable", providerId, reason, @@ -233,6 +204,7 @@ export function composeSqlNamespaceCompletion( if (response.status === "loading") { return Object.freeze({ source: Object.freeze({ + feature: "namespace-catalog", outcome: "loading", providerId: input.outcome.providerId, }), @@ -242,6 +214,7 @@ export function composeSqlNamespaceCompletion( if (response.status === "failed") { return Object.freeze({ source: Object.freeze({ + feature: "namespace-catalog", outcome: "failed", providerId: input.outcome.providerId, }), @@ -285,6 +258,7 @@ export function composeSqlNamespaceCompletion( return Object.freeze({ source: Object.freeze({ coverage: response.coverage, + feature: "namespace-catalog", outcome: "ready", providerId: input.outcome.providerId, }), diff --git a/src/vnext/relation-completion-types.ts b/src/vnext/relation-completion-types.ts index 0dcccaf..f5864d5 100644 --- a/src/vnext/relation-completion-types.ts +++ b/src/vnext/relation-completion-types.ts @@ -258,6 +258,14 @@ export interface SqlColumnCompletionProvenance { readonly columnEntityId: string; } +export interface SqlNamespaceCompletionProvenance { + readonly containerEntityId: string; + readonly epoch: SqlCatalogEpoch; + readonly kind: "namespace-catalog"; + readonly providerId: string; + readonly scope: string; +} + interface SqlCompletionItemBase { readonly label: string; readonly edit: SqlTextChange; @@ -280,6 +288,11 @@ export type SqlCompletionItem = readonly kind: "column"; readonly provenance: SqlColumnCompletionProvenance; readonly relationRequestKey: string; + }) + | (SqlCompletionItemBase & { + readonly kind: "namespace"; + readonly provenance: SqlNamespaceCompletionProvenance; + readonly role: SqlCatalogContainerRole; }); export type SqlCompletionIssue = @@ -301,6 +314,11 @@ export type SqlCompletionIssue = | "column-catalog-malformed" | "column-catalog-partial" | "cte-scope-uncertainty" + | "namespace-catalog-failed" + | "namespace-catalog-loading" + | "namespace-catalog-malformed" + | "namespace-catalog-partial" + | "namespace-prefix-uncertain" | "query-binding-partial" | "query-site-recovery" | "opaque-template-context" @@ -393,9 +411,38 @@ export type SqlColumnCatalogProviderReport = | "provider-failed"; }; +export type SqlNamespaceCatalogProviderReport = + | { + readonly coverage: "complete" | "partial"; + readonly feature: "namespace-catalog"; + readonly outcome: "ready"; + readonly providerId: string; + } + | { + readonly feature: "namespace-catalog"; + readonly outcome: "loading"; + readonly providerId: string; + } + | { + readonly feature: "namespace-catalog"; + readonly outcome: "failed"; + readonly providerId: string; + } + | { + readonly feature: "namespace-catalog"; + readonly outcome: "unavailable"; + readonly providerId: string; + readonly reason: + | "disposed" + | "invalid-request" + | "malformed-response" + | "provider-failed"; + }; + export type SqlCompletionProviderReport = | SqlCatalogProviderReport - | SqlColumnCatalogProviderReport; + | SqlColumnCatalogProviderReport + | SqlNamespaceCatalogProviderReport; export interface SqlServiceFailure { readonly code: "internal"; diff --git a/src/vnext/session.ts b/src/vnext/session.ts index cafecb2..cc16f32 100644 --- a/src/vnext/session.ts +++ b/src/vnext/session.ts @@ -20,6 +20,21 @@ import { composeSqlColumnCompletion, prepareSqlColumnCatalogRelations, } from "./column-completion.js"; +import { + createSqlNamespaceCatalogCoordinator, + type SqlNamespaceCatalogCoordinator, + type SqlNamespaceCatalogOwner, + type SqlNamespaceCatalogSearchOutcome, + type SqlNamespaceCatalogSearchTicket, +} from "./namespace-catalog-coordinator.js"; +import { + composeSqlNamespaceCompletion, + prepareSqlNamespaceCatalogSearch, + type SqlNamespaceCompletionComposition, +} from "./namespace-completion.js"; +import { + MAX_NAMESPACE_RESULTS, +} from "./namespace-catalog-boundary.js"; import { buildSqlStatementIndex, findSqlStatementSlot, @@ -59,6 +74,9 @@ import { createSqlCompletionRefreshToken, type SqlCompletionRequest, type SqlCompletionRefreshToken, + type SqlCompletionIssue, + type SqlCompletionItem, + type SqlCompletionList, type SqlDisposable, type SqlCompletionResult, type SqlCompletionTask, @@ -118,6 +136,7 @@ interface CompletionRequestState { ticket: | SqlCatalogSearchWorkTicket | SqlColumnCatalogBatchTicket + | SqlNamespaceCatalogSearchTicket | null; readonly token: SqlCompletionRefreshToken; } @@ -545,6 +564,68 @@ function completionCancellationReason( return request.cancelReason ?? "superseded"; } +function namespaceCompletionList( + composition: SqlNamespaceCompletionComposition, +): SqlCompletionList { + const items: readonly SqlCompletionItem[] = + composition.value.items.map((item) => + Object.freeze({ + ...(item.detail === undefined ? {} : { detail: item.detail }), + edit: item.edit, + kind: "namespace" as const, + label: item.label, + provenance: item.provenance, + role: item.role, + }) + ); + const issues: readonly SqlCompletionIssue[] = + composition.value.issues.map((reason) => + Object.freeze({ reason }) + ); + const first = issues[0]; + if (first === undefined) { + return Object.freeze({ + isIncomplete: false, + issues: Object.freeze([] as const), + items: Object.freeze(items), + }); + } + const incompleteIssues: [ + SqlCompletionIssue, + ...SqlCompletionIssue[], + ] = [first, ...issues.slice(1)]; + return Object.freeze({ + isIncomplete: true, + issues: Object.freeze(incompleteIssues), + items: Object.freeze(items), + }); +} + +function mergeCompletionLists( + left: SqlCompletionList, + right: SqlCompletionList, +): SqlCompletionList { + const items = Object.freeze([...left.items, ...right.items]); + const issues = Object.freeze([...left.issues, ...right.issues]); + const first = issues[0]; + if (first === undefined) { + return Object.freeze({ + isIncomplete: false, + issues: Object.freeze([] as const), + items, + }); + } + const incompleteIssues: [ + SqlCompletionIssue, + ...SqlCompletionIssue[], + ] = [first, ...issues.slice(1)]; + return Object.freeze({ + isIncomplete: true, + issues: Object.freeze(incompleteIssues), + items, + }); +} + interface MissingDataProperty { readonly found: false; } @@ -952,6 +1033,7 @@ export class DefaultSqlDocumentSession readonly #catalogCoordinator: SqlCatalogSearchWorkCoordinator | null; readonly #catalogResponseBudgetMs: number; readonly #columnCoordinator: SqlColumnCatalogBatchCoordinator | null; + readonly #namespaceCoordinator: SqlNamespaceCatalogCoordinator | null; readonly #dialects: ReadonlyMap; readonly #onDispose: () => void; readonly #listeners = new Set(); @@ -962,6 +1044,9 @@ export class DefaultSqlDocumentSession #columnOwner: SqlColumnCatalogBatchOwner | null = null; #columnOwnerDialect: SqlRelationDialectRuntime | null = null; #columnOwnerScope: string | null = null; + #namespaceOwner: SqlNamespaceCatalogOwner | null = null; + #namespaceOwnerDialect: SqlRelationDialectRuntime | null = null; + #namespaceOwnerScope: string | null = null; #disposed = false; #localRelationStatementCache: | LocalRelationStatementCache @@ -979,11 +1064,13 @@ export class DefaultSqlDocumentSession dialects: ReadonlyMap, catalogCoordinator: SqlCatalogSearchWorkCoordinator | null, columnCoordinator: SqlColumnCatalogBatchCoordinator | null, + namespaceCoordinator: SqlNamespaceCatalogCoordinator | null, completion: CompletionConfiguration, onDispose: () => void, ) { this.#catalogCoordinator = catalogCoordinator; this.#columnCoordinator = columnCoordinator; + this.#namespaceCoordinator = namespaceCoordinator; this.#catalogResponseBudgetMs = completion.catalogResponseBudgetMs; this.#dialects = dialects; @@ -1005,6 +1092,7 @@ export class DefaultSqlDocumentSession }); this.#replaceCatalogOwner(); this.#replaceColumnOwner(); + this.#replaceNamespaceOwner(); } get revision(): SqlRevision { @@ -1289,6 +1377,34 @@ export class DefaultSqlDocumentSession this.#columnOwnerScope = catalog.scope; } + #replaceNamespaceOwner(): void { + const catalog = resolveCatalogContext(this.#snapshot.context); + const dialect = this.#snapshot.dialect.relationDialect; + if ( + this.#namespaceOwner && + catalog && + this.#namespaceOwnerScope === catalog.scope && + this.#namespaceOwnerDialect === dialect + ) { + return; + } + this.#namespaceOwner?.dispose(); + this.#namespaceOwner = null; + this.#namespaceOwnerDialect = null; + this.#namespaceOwnerScope = null; + if (!catalog || !this.#namespaceCoordinator || this.#disposed) { + return; + } + const prepared = this.#namespaceCoordinator.prepareOwner({ + dialectId: dialect.id, + scope: catalog.scope, + }); + if (prepared.status !== "prepared") return; + this.#namespaceOwner = prepared.owner; + this.#namespaceOwnerDialect = dialect; + this.#namespaceOwnerScope = catalog.scope; + } + readonly onDidChange = ( listener: (event: SqlSessionChangeEvent) => void, ): SqlDisposable => { @@ -1942,6 +2058,98 @@ export class DefaultSqlDocumentSession cancelPrevious(); } + let namespaceComposition: SqlNamespaceCompletionComposition | null = + null; + const namespaceOwner = this.#namespaceOwner; + const namespaceCoordinator = this.#namespaceCoordinator; + if (catalog && namespaceOwner && namespaceCoordinator) { + const ticket = namespaceOwner.request( + prepareSqlNamespaceCatalogSearch( + { + prefix: querySite.prefix, + qualifier: querySite.qualifier, + replacementRange, + }, + null, + catalog.searchPaths, + MAX_NAMESPACE_RESULTS, + ), + ); + active.ticket = ticket; + const elapsed = Math.max( + 0, + performance.now() - completionStartedAt, + ); + const remaining = Math.max( + 0, + this.#catalogResponseBudgetMs - elapsed, + ); + let responseTimer: + | ReturnType + | undefined; + const raced = await Promise.race([ + ticket.result.then((outcome) => + Object.freeze({ + kind: "outcome" as const, + outcome, + }), + ), + new Promise<{ readonly kind: "timeout" }>((resolve) => { + responseTimer = setTimeout( + () => resolve(Object.freeze({ kind: "timeout" })), + remaining, + ); + }), + ]); + clearTimeout(responseTimer); + if ( + this.#activeCompletion !== active || + active.cancelReason !== null || + snapshot.revision !== this.#snapshot.revision + ) { + ticket.cancel(); + return completionCancellation( + snapshot.revision, + completionCancellationReason(active), + ); + } + if (raced.kind === "timeout") { + ticket.cancel(); + namespaceComposition = Object.freeze({ + source: Object.freeze({ + feature: "namespace-catalog", + outcome: "loading", + providerId: namespaceCoordinator.providerId, + }), + value: Object.freeze({ + isIncomplete: true, + issues: Object.freeze([ + "namespace-catalog-loading" as const, + ] as const), + items: Object.freeze([]), + }), + }); + } else { + const outcome: SqlNamespaceCatalogSearchOutcome = + raced.outcome; + namespaceComposition = composeSqlNamespaceCompletion({ + matchPrefix: + snapshot.dialect.relationDialect.completion + .cteIdentifierMatchesPrefix, + outcome, + prefix: querySite.prefix, + providerId: namespaceCoordinator.providerId, + replacementRange, + }); + if (!namespaceComposition) { + return completionCancellation( + snapshot.revision, + completionCancellationReason(active), + ); + } + } + } + const composition = composeSqlRelationCompletion({ catalogOutcome, dialect: snapshot.dialect.relationDialect, @@ -1964,6 +2172,18 @@ export class DefaultSqlDocumentSession completionCancellationReason(active), ); } + const value = namespaceComposition + ? mergeCompletionLists( + composition.value, + namespaceCompletionList(namespaceComposition), + ) + : composition.value; + const sources = namespaceComposition + ? Object.freeze([ + ...composition.sources, + namespaceComposition.source, + ]) + : composition.sources; return Object.freeze({ refreshToken: (catalogOutcome?.status === "loading" || @@ -1974,9 +2194,9 @@ export class DefaultSqlDocumentSession ? active.token : null, revision: snapshot.revision, - sources: composition.sources, + sources, status: "ready", - value: composition.value, + value, }); } finally { if (this.#activeCompletion === active) { @@ -2197,7 +2417,12 @@ export class DefaultSqlDocumentSession this.#dialects, ); const nextCatalog = resolveCatalogContext(nextContext); - if (nextCatalog && !this.#catalogCoordinator) { + if ( + nextCatalog && + !this.#catalogCoordinator && + !this.#columnCoordinator && + !this.#namespaceCoordinator + ) { throw new SqlSessionError( "invalid-context", "SQL catalog context requires a configured catalog provider", @@ -2290,6 +2515,7 @@ export class DefaultSqlDocumentSession } this.#replaceCatalogOwner(); this.#replaceColumnOwner(); + this.#replaceNamespaceOwner(); return this.#snapshot.revision; } @@ -2306,10 +2532,12 @@ export class DefaultSqlDocumentSession const refreshIntent = this.#refreshIntent; const catalogOwner = this.#catalogOwner; const columnOwner = this.#columnOwner; + const namespaceOwner = this.#namespaceOwner; this.#activeCompletion = null; this.#refreshIntent = null; this.#catalogOwner = null; this.#columnOwner = null; + this.#namespaceOwner = null; this.#clearSoftRefreshIntentTimer(); this.#clearTerminalIntent(); this.#listeners.clear(); @@ -2334,6 +2562,7 @@ export class DefaultSqlDocumentSession } catalogOwner?.dispose(); columnOwner?.dispose(); + namespaceOwner?.dispose(); this.#onDispose(); }; } @@ -2343,6 +2572,7 @@ export class DefaultSqlLanguageService { readonly #catalogCoordinator: SqlCatalogSearchWorkCoordinator | null; readonly #columnCoordinator: SqlColumnCatalogBatchCoordinator | null; + readonly #namespaceCoordinator: SqlNamespaceCatalogCoordinator | null; readonly #completion: CompletionConfiguration; readonly #dialects: ReadonlyMap; readonly #sessions = new Set>(); @@ -2497,6 +2727,27 @@ export class DefaultSqlLanguageService } this.#columnCoordinator = coordinator.coordinator; } + + const namespaces = readOwnDataProperty( + options, + "namespaces", + "invalid-service-options", + "SQL language service options", + ); + if (!namespaces.found || namespaces.value === undefined) { + this.#namespaceCoordinator = null; + } else { + const coordinator = createSqlNamespaceCatalogCoordinator({ + provider: namespaces.value, + }); + if (coordinator.status !== "created") { + throw new SqlSessionError( + "invalid-service-options", + "SQL namespace catalog provider is invalid", + ); + } + this.#namespaceCoordinator = coordinator.coordinator; + } } catch (error) { if (error instanceof SqlSessionError) { throw error; @@ -2559,7 +2810,12 @@ export class DefaultSqlLanguageService const context = cloneContext(candidateContext); resolveDialectRuntime(context, this.#dialects); const catalogContext = resolveCatalogContext(context); - if (catalogContext && !this.#catalogCoordinator) { + if ( + catalogContext && + !this.#catalogCoordinator && + !this.#columnCoordinator && + !this.#namespaceCoordinator + ) { throw new SqlSessionError( "invalid-context", "SQL catalog context requires a configured catalog provider", @@ -2573,6 +2829,7 @@ export class DefaultSqlLanguageService this.#dialects, this.#catalogCoordinator, this.#columnCoordinator, + this.#namespaceCoordinator, this.#completion, () => { this.#sessions.delete(session); @@ -2615,6 +2872,7 @@ export class DefaultSqlLanguageService this.#sessions.clear(); this.#catalogCoordinator?.dispose(); this.#columnCoordinator?.dispose(); + this.#namespaceCoordinator?.dispose(); }; } diff --git a/src/vnext/types.ts b/src/vnext/types.ts index d2055cb..18a3089 100644 --- a/src/vnext/types.ts +++ b/src/vnext/types.ts @@ -167,6 +167,7 @@ export interface SqlLanguageServiceOptions { readonly catalogResponseBudgetMs?: number | undefined; } | undefined; readonly dialects: readonly SqlDialect[]; + readonly namespaces?: SqlNamespaceCatalogProvider | undefined; } export type SqlSessionErrorCode = @@ -196,6 +197,9 @@ export class SqlSessionError extends Error { import type { SqlColumnCatalogProvider, } from "./column-catalog-types.js"; +import type { + SqlNamespaceCatalogProvider, +} from "./namespace-catalog-types.js"; import type { SqlCompletionRequest, SqlCompletionTask, diff --git a/test/vnext-types/marimo-sql-migration.test-d.ts b/test/vnext-types/marimo-sql-migration.test-d.ts index 740e012..8af1a77 100644 --- a/test/vnext-types/marimo-sql-migration.test-d.ts +++ b/test/vnext-types/marimo-sql-migration.test-d.ts @@ -27,6 +27,7 @@ import { type SqlEmbeddedRegion, type SqlIdentifierComponent, type SqlIdentifierPath, + type SqlNamespaceCatalogProvider, type SqlRelationCatalogProvider, } from "../../src/vnext/index.js"; import { @@ -263,6 +264,44 @@ const marimoColumnProvider: SqlColumnCatalogProvider = { }, }; +const marimoNamespaceProvider: SqlNamespaceCatalogProvider = { + id: "marimo-namespaces", + search: async (request, signal) => { + signal.throwIfAborted(); + const projections = + namespaceProjectionByScope.get(request.scope) ?? []; + return { + containers: projections.flatMap((projection) => { + const first = projection.path[0]; + if (first === undefined) return []; + const canonicalPath = [ + { + ...first, + role: projection.kind, + }, + ...projection.path.slice(1).map((component) => ({ + ...component, + role: projection.kind, + })), + ] as const; + return [{ + canonicalPath, + containerEntityId: projection.entityId, + insertText: + (canonicalPath[canonicalPath.length - 1] ?? first).value, + matchQuality: "exact" as const, + }]; + }), + coverage: "complete", + epoch: catalogByScope.get(request.scope)?.epoch ?? { + generation: 0, + token: "missing-scope", + }, + status: "ready", + }; + }, +}; + // One caller-owned service is shared by every SQL editor support/view. const sharedSqlService = createSqlLanguageService({ @@ -274,6 +313,7 @@ const sharedSqlService = duckdbDialect(), postgresDialect(), ], + namespaces: marimoNamespaceProvider, }); const infoResolver: SqlCompletionInfoResolver = ( From 4e613b2951de77338e7ca3f5acd7748b1916c413 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 03:01:18 +0800 Subject: [PATCH 12/25] refactor(vnext): simplify authenticated completion paths --- .../__tests__/local-relation-site.test.ts | 19 ++++++++++ src/vnext/local-relation-site.ts | 7 ---- src/vnext/namespace-completion.ts | 35 ++++++++++--------- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/vnext/__tests__/local-relation-site.test.ts b/src/vnext/__tests__/local-relation-site.test.ts index 899e6c1..22b4425 100644 --- a/src/vnext/__tests__/local-relation-site.test.ts +++ b/src/vnext/__tests__/local-relation-site.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + analyzeSqlLocalColumnSite, analyzeSqlLocalRelationSite, prepareSqlLocalRelationStatement, type SqlLocalRelationSiteResult, @@ -495,6 +496,24 @@ describe("local relation-site evidence", () => { reason: "ambiguous-query-site", status: "unavailable", }); + expect( + Reflect.apply(analyzeSqlLocalColumnSite, undefined, [ + null, + fixture.position, + ]), + ).toEqual({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + expect( + analyzeSqlLocalColumnSite( + { ...preparation.statement }, + fixture.position, + ), + ).toEqual({ + reason: "ambiguous-query-site", + status: "unavailable", + }); }); it("is deterministic and freezes every exposed wrapper", () => { diff --git a/src/vnext/local-relation-site.ts b/src/vnext/local-relation-site.ts index 4fa2c5e..0c09230 100644 --- a/src/vnext/local-relation-site.ts +++ b/src/vnext/local-relation-site.ts @@ -183,13 +183,6 @@ export function analyzeSqlLocalRelationSite( }); } const relativePosition = position - context.slot.source.from; - if ( - !Number.isSafeInteger(relativePosition) || - relativePosition < 0 || - relativePosition > context.layout.statementLength - ) { - return unavailableSite(); - } return Object.freeze({ local: Object.freeze({ cteVisibility: visibleSqlCtesAt( diff --git a/src/vnext/namespace-completion.ts b/src/vnext/namespace-completion.ts index 0c70406..56f0ee0 100644 --- a/src/vnext/namespace-completion.ts +++ b/src/vnext/namespace-completion.ts @@ -3,8 +3,10 @@ import type { SqlNamespaceCatalogSearchOutcome, } from "./namespace-catalog-coordinator.js"; import type { + SqlCanonicalNamespacePath, SqlNamespaceCatalogResolvedContainer, SqlNamespaceContainerRole, + SqlNamespacePathComponent, SqlNamespaceQuerySite, } from "./namespace-catalog-types.js"; import type { @@ -67,8 +69,12 @@ function compareText(left: string, right: string): number { return left < right ? -1 : left > right ? 1 : 0; } -function last(values: readonly Value[]): Value | null { - return values[values.length - 1] ?? null; +function finalComponent( + path: SqlCanonicalNamespacePath, +): SqlNamespacePathComponent { + let current = path[0]; + for (const component of path.slice(1)) current = component; + return current; } function pathText( @@ -83,8 +89,8 @@ function compareContainers( left: SqlNamespaceCatalogResolvedContainer, right: SqlNamespaceCatalogResolvedContainer, ): number { - const leftLast = last(left.canonicalPath); - const rightLast = last(right.canonicalPath); + const leftLast = finalComponent(left.canonicalPath); + const rightLast = finalComponent(right.canonicalPath); return ( (left.matchQuality === right.matchQuality ? 0 @@ -93,11 +99,9 @@ function compareContainers( : 1) || left.canonicalPath.length - right.canonicalPath.length || ( - leftLast && rightLast - ? ROLE_ORDER[leftLast.role] - ROLE_ORDER[rightLast.role] - : 0 + ROLE_ORDER[leftLast.role] - ROLE_ORDER[rightLast.role] ) || - compareText(leftLast?.value ?? "", rightLast?.value ?? "") || + compareText(leftLast.value, rightLast.value) || compareText(pathText(left), pathText(right)) || compareText( left.provenance.containerEntityId, @@ -158,9 +162,8 @@ function unavailable( function item( container: SqlNamespaceCatalogResolvedContainer, replacementRange: SqlTextRange, -): SqlNamespaceCompletionItem | null { - const component = last(container.canonicalPath); - if (!component) return null; +): SqlNamespaceCompletionItem { + const component = finalComponent(container.canonicalPath); return Object.freeze({ ...(container.detail === undefined ? {} @@ -228,8 +231,7 @@ export function composeSqlNamespaceCompletion( const seen = new Set(); const containers: SqlNamespaceCatalogResolvedContainer[] = []; for (const container of response.containers) { - const component = last(container.canonicalPath); - if (!component) continue; + const component = finalComponent(container.canonicalPath); let match: ReturnType; try { match = input.matchPrefix(component, input.prefix); @@ -251,10 +253,9 @@ export function composeSqlNamespaceCompletion( containers.push(container); } containers.sort(compareContainers); - const items = containers.flatMap((container) => { - const value = item(container, input.replacementRange); - return value ? [value] : []; - }); + const items = containers.map((container) => + item(container, input.replacementRange) + ); return Object.freeze({ source: Object.freeze({ coverage: response.coverage, From 06a02ebbfd1ae135b72b339ef4fd49142abd474c Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 03:04:09 +0800 Subject: [PATCH 13/25] test(vnext): harden integrated completion paths --- src/vnext/__tests__/session.test.ts | 62 +++++++++++++++ .../codemirror/__tests__/sql-editor.test.ts | 79 +++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index 0507d4a..c6a49d1 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -1570,6 +1570,56 @@ describe("namespace completion session integration", () => { vi.useRealTimers(); } }); + + it("supports a namespace-only service and preserves partial evidence", async () => { + const service = createSqlLanguageService({ + dialects: [duckdb], + namespaces: { + id: "namespace-only", + search: async () => ({ + containers: [{ + canonicalPath: [{ + quoted: false, + role: "schema", + value: "main", + }], + containerEntityId: "schema:main", + insertText: "main", + matchQuality: "equivalent", + }], + coverage: "partial", + epoch: { generation: 1, token: "epoch-1" }, + status: "ready", + }), + }, + }); + const text = "SELECT * FROM m"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:namespace-only" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + + await expect(session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + sources: [{ + coverage: "partial", + feature: "namespace-catalog", + }], + status: "ready", + value: { + isIncomplete: true, + issues: [{ reason: "namespace-catalog-partial" }], + items: [{ kind: "namespace", label: "main" }], + }, + }); + service.dispose(); + }); }); describe("statement-index session cache", () => { @@ -3883,6 +3933,18 @@ describe("session coverage hardening", () => { }); }); + it.each([ + { columns: { id: "", loadColumns: async () => ({}) } }, + { namespaces: { id: "", search: async () => ({}) } }, + ])("rejects a malformed auxiliary provider %#", (provider) => { + expectSessionError("invalid-service-options", () => { + createSqlLanguageService({ + ...provider, + dialects: [duckdb], + }); + }); + }); + it.each([ { expectedIssue: "catalog-failed", diff --git a/src/vnext/codemirror/__tests__/sql-editor.test.ts b/src/vnext/codemirror/__tests__/sql-editor.test.ts index 9e4aa4b..220b648 100644 --- a/src/vnext/codemirror/__tests__/sql-editor.test.ts +++ b/src/vnext/codemirror/__tests__/sql-editor.test.ts @@ -1096,6 +1096,32 @@ describe("sqlEditor", () => { expect(harness.sessionDisposals()).toBe(0); }); + it("refuses a completion edit that overlaps an embedded region", async () => { + const harness = fakeService((revision) => + readyResult(revision, [completionItem(14, 16)]) + ); + const support = sqlEditor({ + initialContext: context(), + initialEmbeddedRegions: [{ + from: 15, + language: "host", + to: 16, + }], + service: harness.service, + }); + const view = createView(support.extension); + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + const completion = currentCompletions(view.state)[0]; + if (!completion || typeof completion.apply !== "function") { + throw new Error("Expected completion apply callback"); + } + + completion.apply(view, completion, 14, 16); + expect(view.state.doc.toString()).toBe("SELECT * FROM us"); + expect(harness.updates).toEqual([]); + }); + it("combines document and final context effects into one current input", async () => { const requests: SqlCatalogSearchRequest[] = []; const service = createSqlLanguageService({ @@ -1275,6 +1301,59 @@ describe("sqlEditor", () => { ); }); + it("maps column and namespace completion presentation", async () => { + const epoch = { generation: 1, token: "epoch-1" }; + const items: readonly SqlCompletionItem[] = [{ + edit: { from: 14, insert: "users_column", to: 16 }, + kind: "column", + label: "users_column", + provenance: { + columnEntityId: "users:name", + epoch, + kind: "column-catalog", + providerId: "columns", + relationEntityId: "users", + scope: "connection:fake", + }, + relationRequestKey: "binding:0", + }, { + edit: { from: 14, insert: "users_namespace", to: 16 }, + kind: "namespace", + label: "users_namespace", + provenance: { + containerEntityId: "schema:main", + epoch, + kind: "namespace-catalog", + providerId: "namespaces", + scope: "connection:fake", + }, + role: "schema", + }]; + const harness = fakeService((revision) => + readyResult(revision, items) + ); + const support = sqlEditor({ + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + expect(currentCompletions(view.state)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + label: "users_column", + type: "property", + }), + expect.objectContaining({ + label: "users_namespace", + type: "namespace", + }), + ]), + ); + }); + it("denies SQL completion at an unmatched template EOF without removing external sources", async () => { const harness = fakeService((revision) => readyResult(revision, [completionItem()]) From dd5a8f3dbb18e147e22aedb5038ce83bebb5f3d7 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 03:05:01 +0800 Subject: [PATCH 14/25] test(vnext): enforce namespace coverage floor --- .../namespace-catalog-boundary.test.ts | 64 +++++++++++++++++++ .../namespace-catalog-coordinator.test.ts | 57 +++++++++++++---- src/vnext/namespace-catalog-coordinator.ts | 12 ++-- 3 files changed, 114 insertions(+), 19 deletions(-) diff --git a/src/vnext/__tests__/namespace-catalog-boundary.test.ts b/src/vnext/__tests__/namespace-catalog-boundary.test.ts index 4bee715..126695a 100644 --- a/src/vnext/__tests__/namespace-catalog-boundary.test.ts +++ b/src/vnext/__tests__/namespace-catalog-boundary.test.ts @@ -61,6 +61,11 @@ describe("namespace catalog boundary", () => { expect(context?.id).toBe("namespaces"); context?.search(request(), new AbortController().signal); expect(receivers).toEqual([undefined]); + expect(Reflect.apply( + resolveSqlNamespaceCatalogProvider, + undefined, + [{}], + )).toBeNull(); expect(Object.isFrozen(handle)).toBe(true); }); @@ -136,6 +141,24 @@ describe("namespace catalog boundary", () => { const accessor = { ...valid }; Object.defineProperty(accessor, "prefix", { get: vi.fn() }); cases.push(accessor); + cases.push({ + ...valid, + qualifier: new Proxy([], { + getOwnPropertyDescriptor: (_target, key) => { + if (key === "length") throw new Error("length trap"); + return undefined; + }, + }), + }); + cases.push({ + ...valid, + qualifier: new Proxy([{ quoted: false, value: "x" }], { + getOwnPropertyDescriptor: (target, key) => { + if (key === "0") throw new Error("element trap"); + return Reflect.getOwnPropertyDescriptor(target, key); + }, + }), + }); for (const value of cases) { expect(createSqlNamespaceCatalogSearchRequest(value).status) .toBe("malformed"); @@ -208,6 +231,41 @@ describe("namespace catalog boundary", () => { }); }); + it("orders tied identities and accepts quoted paths without detail", () => { + const provider = captured(); + const base = { + canonicalPath: [{ + quoted: true, + role: "schema", + value: "Main", + }], + insertText: "\"Main\"", + matchQuality: "exact", + }; + const decoded = decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + { + containers: [ + { ...base, containerEntityId: "z" }, + { ...base, containerEntityId: "a" }, + ], + coverage: "complete", + epoch, + status: "ready", + }, + ); + expect(decoded).toMatchObject({ + status: "accepted", + value: { + containers: [ + { containerEntityId: "a" }, + { containerEntityId: "z" }, + ], + }, + }); + }); + it("rejects malformed, conflicting, stale, and oversized responses", () => { const provider = captured(); const valid = { @@ -245,6 +303,7 @@ describe("namespace catalog boundary", () => { (_, index) => container(String(index), "schema", String(index)), ) }, conflicting, + { ...valid, containers: [null] }, { epoch, extra: true, status: "loading" }, { code: "bad", epoch, retry: "never", status: "failed" }, { code: "unknown", epoch, retry: "bad", status: "failed" }, @@ -261,5 +320,10 @@ describe("namespace catalog boundary", () => { value, ).status).toBe("malformed"); } + expect(Reflect.apply( + decodeSqlNamespaceCatalogSearchResponse, + undefined, + [{}, request(), valid], + )).toMatchObject({ status: "malformed" }); }); }); diff --git a/src/vnext/__tests__/namespace-catalog-coordinator.test.ts b/src/vnext/__tests__/namespace-catalog-coordinator.test.ts index 7f1f062..c4f0d76 100644 --- a/src/vnext/__tests__/namespace-catalog-coordinator.test.ts +++ b/src/vnext/__tests__/namespace-catalog-coordinator.test.ts @@ -8,6 +8,7 @@ import { prepareSqlNamespaceCatalogSearch, } from "../namespace-completion.js"; import type { + SqlNamespaceCatalogResolvedContainer, SqlNamespaceCatalogSearchRequest, } from "../namespace-catalog-types.js"; import type { SqlCatalogEpoch } from "../relation-completion-types.js"; @@ -91,12 +92,15 @@ describe("namespace catalog coordinator", () => { searches.push(request); return response(request); }); - const outcome = await owner.request(input()).result; + const outcome = await owner.request({ + ...input(), + prefix: { quoted: true, value: "ma" }, + }).result; expect(searches).toHaveLength(1); expect(searches[0]).toMatchObject({ dialectId: "duckdb", limit: 10, - prefix: { value: "ma" }, + prefix: { quoted: true, value: "ma" }, qualifier: [{ value: "memory" }], scope: "connection", }); @@ -169,7 +173,7 @@ describe("namespace catalog coordinator", () => { }); expect(signals[1]?.aborted).toBe(true); pending[0]?.resolve({}); - pending[1]?.resolve({}); + pending[1]?.reject(new Error("late")); await Promise.resolve(); }); @@ -188,6 +192,7 @@ describe("namespace catalog coordinator", () => { const first = owner.request(input("first")); const second = other.owner.request(input("second")); owner.dispose(); + owner.dispose(); await expect(first.result).resolves.toEqual({ reason: "disposed", status: "unavailable", @@ -199,6 +204,7 @@ describe("namespace catalog coordinator", () => { await Promise.resolve(); expect(secondSettled).toBe(false); coordinator.dispose(); + coordinator.dispose(); await expect(second.result).resolves.toEqual({ reason: "disposed", status: "unavailable", @@ -276,10 +282,12 @@ describe("namespace catalog coordinator", () => { scope: "scope", }); if (prepared.status !== "prepared") throw new Error("prepared"); - expect(await prepared.owner.request({ + const invalidTicket = prepared.owner.request({ ...input(), limit: 0, - }).result).toMatchObject({ + }); + invalidTicket.cancel(); + expect(await invalidTicket.result).toMatchObject({ reason: "invalid-request", status: "unavailable", }); @@ -372,6 +380,29 @@ describe("namespace completion composer", () => { const catalog = catalogResponse(); const main = catalog.containers[0]; if (!main) throw new Error("main"); + const parent = main.canonicalPath[0]; + const child = main.canonicalPath[1]; + if (!child) throw new Error("child"); + const copied: SqlNamespaceCatalogResolvedContainer = { + ...main, + canonicalPath: [ + parent, + { ...child, quoted: true }, + ], + containerEntityId: "main-copy", + provenance: { + ...main.provenance, + containerEntityId: "main-copy", + }, + }; + const tied: SqlNamespaceCatalogResolvedContainer = { + ...main, + containerEntityId: "aaa", + provenance: { + ...main.provenance, + containerEntityId: "aaa", + }, + }; const duplicated = { ...readyOutcome, response: { @@ -379,14 +410,8 @@ describe("namespace completion composer", () => { containers: [ ...catalog.containers, main, - { - ...main, - containerEntityId: "main-copy", - provenance: { - ...main.provenance, - containerEntityId: "main-copy", - }, - }, + copied, + tied, ], }, }; @@ -398,7 +423,7 @@ describe("namespace completion composer", () => { replacementRange: { from: 10, to: 12 }, }); expect(all?.value.items.map((value) => value.label)) - .toEqual(["main", "main", "metrics"]); + .toEqual(["main", "main", "main", "metrics"]); const composition = composeSqlNamespaceCompletion({ matchPrefix: (candidate, prefix) => candidate.value.startsWith(prefix.value) @@ -421,6 +446,10 @@ describe("namespace completion composer", () => { edit: { from: 10, insert: "main", to: 12 }, label: "main", role: "schema", + }, { + edit: { from: 10, insert: "main", to: 12 }, + label: "main", + role: "schema", }], }, }); diff --git a/src/vnext/namespace-catalog-coordinator.ts b/src/vnext/namespace-catalog-coordinator.ts index d43ba13..7b7d71d 100644 --- a/src/vnext/namespace-catalog-coordinator.ts +++ b/src/vnext/namespace-catalog-coordinator.ts @@ -356,18 +356,20 @@ function request( status: "usable", })); } - let resolveResult: - (value: SqlNamespaceCatalogSearchOutcome) => void = - (): void => {}; + const resolver: { + value: + | ((value: SqlNamespaceCatalogSearchOutcome) => void) + | null; + } = { value: null }; const result = new Promise( (resolve) => { - resolveResult = resolve; + resolver.value = resolve; }, ); const consumer: ConsumerState = { controller: new AbortController(), owner, - resolve: resolveResult, + resolve: resolver.value, settled: false, }; owner.active = consumer; From fd249bd8b2203afcef90fff80a6df2a12063dd8b Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 03:06:04 +0800 Subject: [PATCH 15/25] test(vnext): harden column catalog boundaries --- .../column-catalog-batch-coordinator.test.ts | 67 +++- .../__tests__/column-catalog-boundary.test.ts | 315 ++++++++++++++++++ src/vnext/column-catalog-batch-coordinator.ts | 86 ++--- 3 files changed, 411 insertions(+), 57 deletions(-) diff --git a/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts index 174f0cc..8920fe1 100644 --- a/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts +++ b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts @@ -152,6 +152,30 @@ describe("column catalog batch coordinator", () => { }); }); + it("separates quoted relation and search-path cache identities", async () => { + let calls = 0; + const { owner } = setup((request) => { + calls += 1; + return ready(request); + }); + const quoted = { + expectedEpoch: epoch, + relations: [{ + path: [{ quoted: true, value: "users" }], + requestKey: "quoted", + }], + searchPaths: [[{ quoted: true, value: "main" }]], + }; + + await expect(owner.request(quoted).result).resolves.toMatchObject({ + status: "usable", + }); + await expect(owner.request(quoted).result).resolves.toMatchObject({ + status: "usable", + }); + expect(calls).toBe(1); + }); + it("supports cold null epochs and reuses the observed response epoch", async () => { const expected: Array = []; const { owner } = setup((request) => { @@ -262,6 +286,17 @@ describe("column catalog batch coordinator", () => { await expect(ticket.result).resolves.toEqual({ status: "cancelled" }); }); + it("ignores a provider rejection after its consumer is cancelled", async () => { + const work = deferred(); + const { owner } = setup(() => work.promise); + const ticket = owner.request(input()); + + ticket.cancel(); + work.reject(new Error("late rejection")); + await expect(ticket.result).resolves.toEqual({ status: "cancelled" }); + await Promise.resolve(); + }); + it("supersedes prior owner work but isolates other owners", async () => { const pending: Array>> = []; const { coordinator, owner } = setup(() => { @@ -301,6 +336,7 @@ describe("column catalog batch coordinator", () => { }); const ownerTicket = owner.request(input()); owner.dispose(); + owner.dispose(); await expect(ownerTicket.result).resolves.toEqual({ reason: "disposed", status: "unavailable", @@ -376,6 +412,12 @@ describe("column catalog batch coordinator", () => { reason: "invalid-provider", status: "unavailable", }); + expect(createSqlColumnCatalogBatchCoordinator({ + provider: {}, + })).toEqual({ + reason: "invalid-provider", + status: "unavailable", + }); expect(createSqlColumnCatalogBatchCoordinator({ maxCacheEntries: 0, provider: { id: "catalog", loadColumns: vi.fn() }, @@ -396,10 +438,33 @@ describe("column catalog batch coordinator", () => { scope: "scope", }); if (prepared.status !== "prepared") throw new Error("Expected owner"); - await expect(prepared.owner.request({ + const invalidTicket = prepared.owner.request({ expectedEpoch: epoch, relations: [], searchPaths: [], + }); + invalidTicket.cancel(); + await expect(invalidTicket.result).resolves.toEqual({ + reason: "invalid-request", + status: "unavailable", + }); + }); + + it("contains hostile and missing request properties", async () => { + const { owner } = setup(vi.fn()); + const hostile = new Proxy({}, { + getOwnPropertyDescriptor() { + throw new Error("hostile descriptor"); + }, + }); + + await expect(owner.request(hostile).result).resolves.toEqual({ + reason: "invalid-request", + status: "unavailable", + }); + await expect(owner.request({ + expectedEpoch: epoch, + relations: [reference("users")], }).result).resolves.toEqual({ reason: "invalid-request", status: "unavailable", diff --git a/src/vnext/__tests__/column-catalog-boundary.test.ts b/src/vnext/__tests__/column-catalog-boundary.test.ts index b067603..965c724 100644 --- a/src/vnext/__tests__/column-catalog-boundary.test.ts +++ b/src/vnext/__tests__/column-catalog-boundary.test.ts @@ -127,6 +127,31 @@ describe("column catalog provider boundary", () => { loadColumns: () => undefined, }).status).toBe("malformed"); expect(captureSqlColumnCatalogProvider(null).status).toBe("malformed"); + expect(resolveSqlColumnCatalogProvider(null)).toBeNull(); + expect(resolveSqlColumnCatalogProvider(1)).toBeNull(); + }); + + it("contains hostile own-key and descriptor traps", () => { + const ownKeys = new Proxy( + { id: "catalog", loadColumns: () => undefined }, + { + ownKeys() { + throw new Error("private ownKeys trap"); + }, + }, + ); + const descriptor = new Proxy( + { id: "catalog", loadColumns: () => undefined }, + { + getOwnPropertyDescriptor() { + throw new Error("private descriptor trap"); + }, + }, + ); + expect(captureSqlColumnCatalogProvider(ownKeys).status).toBe("malformed"); + expect(captureSqlColumnCatalogProvider(descriptor).status).toBe( + "malformed", + ); }); }); @@ -239,6 +264,99 @@ describe("column catalog batch request boundary", () => { searchPaths: [], }).status).toBe("accepted"); }); + + it("rejects malformed epochs, path components, search paths, and array traps", () => { + const base = { + dialectId: "duckdb", + expectedEpoch: epoch, + relations: [{ path, requestKey: "users" }], + scope: "scope", + searchPaths: [], + }; + const badEpochs = [ + { generation: -1, token: "x" }, + { generation: 1.5, token: "x" }, + { generation: 1, token: "" }, + { generation: "1", token: "x" }, + ]; + for (const expectedEpoch of badEpochs) { + expect( + createSqlColumnCatalogBatchRequest({ ...base, expectedEpoch }).status, + ).toBe("malformed"); + } + + const sparsePath: unknown[] = []; + sparsePath.length = 1; + const sparseSearchPaths: unknown[] = []; + sparseSearchPaths.length = 1; + const malformedPaths = [ + sparsePath, + [null], + [{ quoted: "false", value: "users" }], + [{ quoted: false, value: "" }], + ]; + for (const malformedPath of malformedPaths) { + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + relations: [{ path: malformedPath, requestKey: "users" }], + }).status, + ).toBe("malformed"); + } + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + searchPaths: sparseSearchPaths, + }).status, + ).toBe("malformed"); + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + searchPaths: [["not-a-component"]], + }).status, + ).toBe("malformed"); + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + relations: [null], + }).status, + ).toBe("malformed"); + + const trappedLength = new Proxy( + [{ path, requestKey: "users" }], + { + getOwnPropertyDescriptor(target, key) { + if (key === "length") { + throw new Error("private length trap"); + } + return Reflect.getOwnPropertyDescriptor(target, key); + }, + }, + ); + const trappedElement = new Proxy( + [{ path, requestKey: "users" }], + { + getOwnPropertyDescriptor(target, key) { + if (key === "0") { + throw new Error("private element trap"); + } + return Reflect.getOwnPropertyDescriptor(target, key); + }, + }, + ); + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + relations: trappedLength, + }).status, + ).toBe("malformed"); + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + relations: trappedElement, + }).status, + ).toBe("malformed"); + }); }); describe("column catalog response boundary", () => { @@ -469,4 +587,201 @@ describe("column catalog response boundary", () => { ); expect(invalid.status).toBe("malformed"); }); + + it("rejects malformed relation and column state combinations", () => { + const captured = provider(() => undefined); + const request_ = request(); + const validUsers = readyResponse().relations[0]; + const validEvents = readyResponse().relations[1]; + const invalidUsers = [ + null, + { requestKey: "", status: "loading" }, + { + columns: [], + coverage: "complete", + relationEntityId: "", + requestKey: "users", + status: "ready", + }, + { code: "mystery", requestKey: "users", retry: "never", status: "failed" }, + { + code: "unavailable", + requestKey: "users", + retry: "mystery", + status: "failed", + }, + { requestKey: "users", status: "mystery" }, + { + columns: [null], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + { + columns: [{ + columnEntityId: "", + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + { + columns: [{ + columnEntityId: "id", + identifier: null, + insertText: "id", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + { + columns: [{ + columnEntityId: "id", + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: -1, + }], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + ]; + for (const users of invalidUsers) { + expect( + decodeSqlColumnCatalogBatchResponse(captured, request_, { + epoch, + relations: [users, validEvents], + }).status, + ).toBe("malformed"); + } + + const sparseColumns: unknown[] = []; + sparseColumns.length = 1; + const sparseRelations: unknown[] = []; + sparseRelations.length = 2; + expect( + decodeSqlColumnCatalogBatchResponse(captured, request_, { + epoch, + relations: [ + { ...validUsers, columns: sparseColumns }, + validEvents, + ], + }).status, + ).toBe("malformed"); + expect( + decodeSqlColumnCatalogBatchResponse(captured, request_, { + epoch, + relations: sparseRelations, + }).status, + ).toBe("malformed"); + }); + + it("rejects an unexpected relation without conflating it with response size", () => { + const result = decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + { + epoch, + relations: [ + readyResponse().relations[0], + { requestKey: "other", status: "loading" }, + ], + }, + ); + expect(result).toEqual({ + reason: "unexpected-relation", + status: "malformed", + }); + }); + + it("sorts stable ties and enforces the aggregate batch column cap", () => { + const tied = { + epoch, + relations: [ + { + columns: [ + { + columnEntityId: "b", + identifier: { quoted: true, value: "same" }, + insertText: "same", + ordinal: 0, + }, + { + columnEntityId: "a", + identifier: { quoted: true, value: "same" }, + insertText: "same", + ordinal: 0, + }, + ], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + readyResponse().relations[1], + ], + }; + const sorted = decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + tied, + ); + expect(sorted).toMatchObject({ + status: "accepted", + value: { + relations: [ + { requestKey: "events" }, + { + columns: [ + { columnEntityId: "a" }, + { columnEntityId: "b" }, + ], + }, + ], + }, + }); + + const references = Array.from({ length: 17 }, (_, index) => ({ + path, + requestKey: `r${index}`, + })); + const created = createSqlColumnCatalogBatchRequest({ + dialectId: "duckdb", + expectedEpoch: epoch, + relations: references, + scope: "scope", + searchPaths: [], + }); + if (created.status !== "accepted") { + throw new Error("Expected aggregate-limit request"); + } + const relations = references.map((reference, relationIndex) => ({ + columns: Array.from({ length: MAX_COLUMNS_PER_RELATION }, (_, ordinal) => ({ + columnEntityId: `c${relationIndex}-${ordinal}`, + identifier: { quoted: false, value: `c${ordinal}` }, + insertText: `c${ordinal}`, + ordinal, + })), + coverage: "complete", + relationEntityId: `relation-${relationIndex}`, + requestKey: reference.requestKey, + status: "ready", + })); + expect( + decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + created.value, + { epoch, relations }, + ), + ).toEqual({ reason: "resource-limit", status: "malformed" }); + }); }); diff --git a/src/vnext/column-catalog-batch-coordinator.ts b/src/vnext/column-catalog-batch-coordinator.ts index e00b071..0a4a378 100644 --- a/src/vnext/column-catalog-batch-coordinator.ts +++ b/src/vnext/column-catalog-batch-coordinator.ts @@ -210,9 +210,10 @@ function cacheSet( state.cache.delete(key); state.cache.set(key, value); while (state.cache.size > state.maxCacheEntries) { - const oldest = state.cache.keys().next().value; - if (typeof oldest !== "string") break; - state.cache.delete(oldest); + for (const oldest of state.cache.keys()) { + state.cache.delete(oldest); + break; + } } } @@ -229,11 +230,8 @@ function settle( consumer: ConsumerState, outcome: SqlColumnCatalogBatchOutcome, ): void { - if (consumer.settled) return; consumer.settled = true; - if (consumer.owner.active === consumer) { - consumer.owner.active = null; - } + consumer.owner.active = null; const resolve = consumer.resolve; consumer.resolve = null; resolve?.(outcome); @@ -258,28 +256,26 @@ function makeSettledTicket( } function combineRelations( - request: SqlColumnCatalogBatchRequest, cached: ReadonlyMap, loaded: readonly SqlColumnCatalogRelationResult[], -): readonly SqlColumnCatalogRelationResult[] | null { - const byKey = new Map(); - for (const relation of loaded) byKey.set(relation.requestKey, relation); - const output: SqlColumnCatalogRelationResult[] = []; - for (const reference of request.relations) { - const cachedRelation = cached.get(reference.requestKey); - const relation = cachedRelation - ? withRequestKey(cachedRelation, reference.requestKey) - : byKey.get(reference.requestKey); - if (!relation) return null; - output.push(relation); +): readonly SqlColumnCatalogRelationResult[] { + const output: SqlColumnCatalogRelationResult[] = [...loaded]; + for (const [requestKey, relation] of cached) { + output.push(withRequestKey(relation, requestKey)); } + output.sort((left, right) => + left.requestKey < right.requestKey + ? -1 + : left.requestKey > right.requestKey + ? 1 + : 0 + ); return Object.freeze(output); } function startProviderWork( state: CoordinatorState, owner: OwnerState, - fullRequest: SqlColumnCatalogBatchRequest, missingRequest: SqlColumnCatalogBatchRequest, cached: ReadonlyMap, consumer: ConsumerState, @@ -296,15 +292,7 @@ function startProviderWork( } Promise.resolve(providerResult).then( (value) => { - if ( - consumer.settled || - consumer.controller.signal.aborted || - state.disposed || - owner.disposed || - owner.owner !== state - ) { - return; - } + if (consumer.settled) return; const decoded = decodeSqlColumnCatalogBatchResponse( state.capturedProvider, missingRequest, @@ -332,21 +320,18 @@ function startProviderWork( } } const relations = combineRelations( - fullRequest, cached, decoded.value.relations, ); settle( consumer, - relations - ? Object.freeze({ - providerId: state.context.id, - epoch: decoded.value.epoch, - relations, - scope: owner.scope, - status: "usable", - }) - : unavailable("malformed-response"), + Object.freeze({ + providerId: state.context.id, + epoch: decoded.value.epoch, + relations, + scope: owner.scope, + status: "usable", + }), ); }, () => { @@ -399,31 +384,21 @@ function requestColumns( missing.push(reference); } } - if (missing.length === 0) { - const combined = combineRelations(request, cached, []); - if (request.expectedEpoch === null) { - return makeSettledTicket(unavailable("invalid-request")); - } + if (request.expectedEpoch !== null && missing.length === 0) { return makeSettledTicket( Object.freeze({ epoch: request.expectedEpoch, providerId: state.context.id, - relations: combined ?? Object.freeze([]), + relations: combineRelations(cached, []), scope: owner.scope, status: "usable", }), ); } - const missingCreated = createSqlColumnCatalogBatchRequest({ - dialectId: request.dialectId, - expectedEpoch: request.expectedEpoch, - relations: missing, - scope: request.scope, - searchPaths: request.searchPaths, + const missingRequest: SqlColumnCatalogBatchRequest = Object.freeze({ + ...request, + relations: Object.freeze(missing), }); - if (missingCreated.status === "malformed") { - return makeSettledTicket(unavailable("invalid-request")); - } let resolveResult: ( value: SqlColumnCatalogBatchOutcome, ) => void = (): void => {}; @@ -444,8 +419,7 @@ function requestColumns( startProviderWork( state, owner, - request, - missingCreated.value, + missingRequest, cached, consumer, ); From d9324963fc2f121be0fc0fd90f77a9a9e08666ff Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 03:06:19 +0800 Subject: [PATCH 16/25] test(vnext): retain auxiliary catalog owners --- src/vnext/__tests__/session.test.ts | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index c6a49d1..ef6c1a2 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -1620,6 +1620,47 @@ describe("namespace completion session integration", () => { }); service.dispose(); }); + + it("retains auxiliary owners across unrelated context changes", () => { + const service = createSqlLanguageService({ + columns: { + id: "columns", + loadColumns: async () => ({ + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + }), + }, + dialects: [duckdb], + namespaces: { + id: "namespaces", + search: async () => ({ + containers: [], + coverage: "complete", + epoch: { generation: 1, token: "epoch-1" }, + status: "ready", + }), + }, + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:stable" }, + dialect: "duckdb", + engine: "before", + }, + text: "SELECT 1", + }); + const revision = session.update({ + baseRevision: session.revision, + context: { + catalog: { scope: "connection:stable" }, + dialect: "duckdb", + engine: "after", + }, + }); + + expect(session.revision).toBe(revision); + service.dispose(); + }); }); describe("statement-index session cache", () => { From 91f33bd9ef291bfb20cdf357bbc8bf71e282a968 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 03:07:09 +0800 Subject: [PATCH 17/25] test(vnext): type hostile column requests --- .../column-catalog-batch-coordinator.test.ts | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts index 8920fe1..d4477ac 100644 --- a/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts +++ b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts @@ -458,14 +458,24 @@ describe("column catalog batch coordinator", () => { }, }); - await expect(owner.request(hostile).result).resolves.toEqual({ + const hostileTicket = Reflect.apply( + owner.request, + undefined, + [hostile], + ); + await expect(hostileTicket.result).resolves.toEqual({ reason: "invalid-request", status: "unavailable", }); - await expect(owner.request({ - expectedEpoch: epoch, - relations: [reference("users")], - }).result).resolves.toEqual({ + const missingTicket = Reflect.apply( + owner.request, + undefined, + [{ + expectedEpoch: epoch, + relations: [reference("users")], + }], + ); + await expect(missingTicket.result).resolves.toEqual({ reason: "invalid-request", status: "unavailable", }); From 8d9a52a97d64dd80ec5f21c68af93f0efbad41fb Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 03:09:46 +0800 Subject: [PATCH 18/25] perf(vnext): ratchet completed core budget --- docs/vnext/capability-charter.md | 3 ++- scripts/worker-placement.mjs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/vnext/capability-charter.md b/docs/vnext/capability-charter.md index d04c8dd..68e5fe5 100644 --- a/docs/vnext/capability-charter.md +++ b/docs/vnext/capability-charter.md @@ -239,7 +239,8 @@ count and estimated retained bytes. Provisional compressed bundle budgets: -- Framework-independent core with relation completion: 48 KiB +- Framework-independent core with relation, namespace, and column completion: + 50 KiB - Core plus CodeMirror adapter, excluding parser and dialect data: 75 KiB - Each ordinary dialect module: 25 KiB - Optional `node-sql-parser` chunk: no regression from its recorded baseline diff --git a/scripts/worker-placement.mjs b/scripts/worker-placement.mjs index fb6e692..e453071 100644 --- a/scripts/worker-placement.mjs +++ b/scripts/worker-placement.mjs @@ -35,8 +35,8 @@ const PARSER_MARKERS = [ "tableList", ]; const BIGQUERY_GZIP_LIMIT = 50 * 1024; -const CORE_TOTAL_GZIP_LIMIT = 48 * 1024; -const CORE_TOTAL_RAW_LIMIT = 180 * 1024; +const CORE_TOTAL_GZIP_LIMIT = 50 * 1024; +const CORE_TOTAL_RAW_LIMIT = 184 * 1024; const POSTGRESQL_GZIP_LIMIT = 68 * 1024; const WORKER_TOTAL_GZIP_LIMIT = 160 * 1024; const WORKER_TOTAL_RAW_LIMIT = 700 * 1024; From d8590db7ee8f2e3051a7110327178b22849ea897 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 03:30:43 +0800 Subject: [PATCH 19/25] fix(vnext): close final completion review gaps --- docs/vnext/marimo-sql-migration.md | 5 +- .../column-catalog-batch-coordinator.test.ts | 3 +- .../__tests__/column-catalog-boundary.test.ts | 21 +- src/vnext/__tests__/column-completion.test.ts | 95 ++++- src/vnext/__tests__/column-query-site.test.ts | 25 ++ .../namespace-catalog-coordinator.test.ts | 6 +- src/vnext/__tests__/session.test.ts | 343 +++++++++++++++++- .../sql-editor-completion-gate.test.ts | 4 +- src/vnext/codemirror/sql-editor.ts | 7 +- src/vnext/column-catalog-batch-coordinator.ts | 1 - src/vnext/column-catalog-boundary.ts | 23 +- src/vnext/column-catalog-types.ts | 22 +- src/vnext/column-completion.ts | 28 +- src/vnext/column-query-site.ts | 18 +- src/vnext/index.ts | 4 + src/vnext/local-relation-site.ts | 25 +- src/vnext/namespace-catalog-types.ts | 14 +- src/vnext/namespace-completion.ts | 2 + src/vnext/relation-completion-types.ts | 22 +- src/vnext/session.ts | 175 +++++++-- .../marimo-sql-migration.test-d.ts | 2 - 21 files changed, 753 insertions(+), 92 deletions(-) diff --git a/docs/vnext/marimo-sql-migration.md b/docs/vnext/marimo-sql-migration.md index 48b9aae..66b3a95 100644 --- a/docs/vnext/marimo-sql-migration.md +++ b/docs/vnext/marimo-sql-migration.md @@ -38,8 +38,9 @@ reproduce The same shared service configures one `SqlColumnCatalogProvider`. Each `loadColumns` call forwards the complete relation request array to one DataTable metadata batch operation. It does not fetch once per relation. -Relation IDs are the stable IDs emitted by the relation provider. Column IDs -are stable within that relation and connection incarnation. +The column authority resolves each canonical relation path within the same +scope and search paths as relation completion. Column IDs and returned relation +IDs are stable within that connection incarnation. Each column supplies a canonical identifier and provider-rendered `insertText`; these are intentionally separate so quoted or dialect-sensitive diff --git a/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts index d4477ac..8f2c8c2 100644 --- a/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts +++ b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts @@ -38,8 +38,7 @@ function ready( ordinal: 0, }], coverage: "complete", - relationEntityId: - relation.relationEntityId ?? `relation-${relation.requestKey}`, + relationEntityId: `relation-${relation.requestKey}`, requestKey: relation.requestKey, status: "ready", })), diff --git a/src/vnext/__tests__/column-catalog-boundary.test.ts b/src/vnext/__tests__/column-catalog-boundary.test.ts index 965c724..06a8f8b 100644 --- a/src/vnext/__tests__/column-catalog-boundary.test.ts +++ b/src/vnext/__tests__/column-catalog-boundary.test.ts @@ -22,7 +22,6 @@ function request() { { path, requestKey: "users" }, { path: [{ quoted: false, value: "events" }], - relationEntityId: "relation-events", requestKey: "events", }, ], @@ -162,7 +161,7 @@ describe("column catalog batch request boundary", () => { expectedEpoch: epoch, relations: [ { path, requestKey: "z" }, - { path, relationEntityId: "stable-a", requestKey: "a" }, + { path, requestKey: "a" }, ], scope: "scope", searchPaths: [[{ quoted: true, value: "Main" }]], @@ -171,7 +170,7 @@ describe("column catalog batch request boundary", () => { status: "accepted", value: { relations: [ - { relationEntityId: "stable-a", requestKey: "a" }, + { requestKey: "a" }, { requestKey: "z" }, ], }, @@ -476,22 +475,6 @@ describe("column catalog response boundary", () => { ], }, }, - { - name: "conflicting known entity", - value: { - epoch, - relations: [ - readyResponse().relations[0], - { - columns: [], - coverage: "complete", - relationEntityId: "wrong-events-id", - requestKey: "events", - status: "ready", - }, - ], - }, - }, { name: "conflicting duplicate column", value: { diff --git a/src/vnext/__tests__/column-completion.test.ts b/src/vnext/__tests__/column-completion.test.ts index 4cf0c9d..b30e118 100644 --- a/src/vnext/__tests__/column-completion.test.ts +++ b/src/vnext/__tests__/column-completion.test.ts @@ -200,7 +200,18 @@ describe("column completion", () => { dialect: POSTGRESQL_SQL_RELATION_DIALECT, outcome: usable([ { - columns: [column("id", "users", 0)], + columns: [ + column("id", "users", 0), + Object.freeze({ + ...column("id", "users", 1), + columnEntityId: "users:id-alternate", + insertText: "id_alternate", + provenance: Object.freeze({ + ...column("id", "users", 1).provenance, + columnEntityId: "users:id-alternate", + }), + }), + ], coverage: "partial", relationEntityId: "users", requestKey: "binding:0", @@ -390,6 +401,11 @@ describe("column completion", () => { expect(result).toMatchObject({ sources: [{ coverage: "partial", + failures: [{ + code: "unavailable", + requestKey: "binding:1", + retry: "next-request", + }], outcome: "ready", }], value: { @@ -415,6 +431,81 @@ describe("column completion", () => { providerId: "columns", site: current, })?.sources[0], - ).toMatchObject({ outcome: "failed" }); + ).toMatchObject({ + failures: [{ + code: "unavailable", + requestKey: "binding:0", + retry: "next-request", + }], + outcome: "failed", + }); + }); + + it("uses deterministic code-unit ordering", () => { + const current = site(); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const result = composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([{ + columns: [ + column("ä_value", "users", 0), + column("z_value", "users", 1), + ], + coverage: "complete", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }]), + prepared, + providerId: "columns", + site: current, + }); + + expect(result?.value.items.map((item) => item.label)).toEqual([ + "z_value", + "ä_value", + ]); + }); + + it("uses relation detail to order columns with equal labels", () => { + const current = site(); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const result = composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([ + { + columns: [column("id", "users", 0)], + coverage: "complete", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }, + { + columns: [column("id", "orders", 0)], + coverage: "complete", + relationEntityId: "orders", + requestKey: "binding:1", + status: "ready", + }, + ]), + prepared, + providerId: "columns", + site: current, + }); + + expect(result?.value.items.map((item) => item.detail)).toEqual([ + "VARCHAR — o", + "VARCHAR — u", + ]); + expect(result?.value.items.map((item) => item.edit.insert)).toEqual([ + "id", + "id", + ]); }); }); diff --git a/src/vnext/__tests__/column-query-site.test.ts b/src/vnext/__tests__/column-query-site.test.ts index 6fd9eb1..0f075ef 100644 --- a/src/vnext/__tests__/column-query-site.test.ts +++ b/src/vnext/__tests__/column-query-site.test.ts @@ -111,6 +111,31 @@ describe("recognizeSqlColumnQuerySite", () => { ); }); + it("keeps set-operation arms and JOIN visibility isolated", () => { + const firstArm = ready( + analyze("SELECT | FROM users UNION SELECT x FROM secrets"), + ); + expect(firstArm.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(["users"]); + + const secondArm = ready( + analyze("SELECT x FROM users UNION SELECT | FROM secrets"), + ); + expect(secondArm.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(["secrets"]); + + const joinCondition = ready( + analyze( + "SELECT * FROM users u JOIN orders o ON o.user_id = u.| JOIN payments p ON true", + ), + ); + expect(joinCondition.relations.map((relation) => + relation.alias?.value + )).toEqual(["u", "o"]); + }); + it("supports BigQuery quoted multipart bindings", () => { const result = ready( analyze("SELECT t.| FROM `project.dataset.table` AS t", { diff --git a/src/vnext/__tests__/namespace-catalog-coordinator.test.ts b/src/vnext/__tests__/namespace-catalog-coordinator.test.ts index c4f0d76..b603fee 100644 --- a/src/vnext/__tests__/namespace-catalog-coordinator.test.ts +++ b/src/vnext/__tests__/namespace-catalog-coordinator.test.ts @@ -513,7 +513,11 @@ describe("namespace completion composer", () => { status: "usable", }, })).toMatchObject({ - source: { outcome: "failed" }, + source: { + code: "unknown", + outcome: "failed", + retry: "never", + }, value: { issues: ["namespace-catalog-failed"] }, }); expect(composeSqlNamespaceCompletion({ diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index ef6c1a2..a7a6653 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -1269,6 +1269,8 @@ describe("column completion session integration", () => { >[0][] = []; const service = serviceWithColumns(async (request) => { requests.push(request); + const relation = request.relations[0]; + if (!relation) throw new Error("Expected one relation"); return { epoch: { generation: 1, token: "epoch-1" }, relations: [{ @@ -1281,7 +1283,7 @@ describe("column completion session integration", () => { }], coverage: "complete", relationEntityId: "users", - requestKey: request.relations[0]?.requestKey, + requestKey: relation.requestKey, status: "ready", }], }; @@ -1392,6 +1394,37 @@ describe("column completion session integration", () => { service.dispose(); }); + it("never resolves a visible CTE as a physical relation", async () => { + let calls = 0; + const service = serviceWithColumns(async () => { + calls += 1; + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + }; + }); + const text = + "WITH users AS (SELECT 1 AS local_id) SELECT users. FROM users"; + const position = text.indexOf("users.") + "users.".length; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + + await expect(session.complete({ + position, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + status: "unavailable", + }); + expect(calls).toBe(0); + service.dispose(); + }); + it("keeps slow providers inside the completion response budget", async () => { vi.useFakeTimers(); try { @@ -1410,7 +1443,9 @@ describe("column completion session integration", () => { trigger: { kind: "invoked" }, }); await vi.advanceTimersByTimeAsync(40); - await expect(completion).resolves.toMatchObject({ + const result = await completion; + expect(result).toMatchObject({ + refreshToken: expect.anything(), sources: [{ feature: "column-catalog", outcome: "loading", @@ -1427,6 +1462,159 @@ describe("column completion session integration", () => { vi.useRealTimers(); } }); + + it("emits readiness when a timed-out column batch settles", async () => { + vi.useFakeTimers(); + try { + let resolveBatch: + | ((value: Awaited< + ReturnType + >) => void) + | undefined; + const batch = new Promise< + Awaited> + >((resolve) => { + resolveBatch = resolve; + }); + const service = createSqlLanguageService({ + columns: { id: "columns", loadColumns: () => batch }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:columns-ready" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const events: SqlSessionChangeEvent[] = []; + session.onDidChange((event) => events.push(event)); + const pending = session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + await vi.advanceTimersByTimeAsync(0); + const result = await pending; + if (result.status !== "ready" || result.refreshToken === null) { + throw new Error("Expected retained column loading"); + } + + resolveBatch?.({ + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + }); + await Promise.resolve(); + await Promise.resolve(); + expect(events).toMatchObject([{ + reason: "catalog-availability", + refreshToken: result.refreshToken, + }]); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); + + it("emits readiness when a timed-out column batch rejects", async () => { + vi.useFakeTimers(); + try { + let rejectBatch: ((reason: Error) => void) | undefined; + const batch = new Promise< + Awaited> + >((_resolve, reject) => { + rejectBatch = reject; + }); + const service = createSqlLanguageService({ + columns: { id: "columns", loadColumns: () => batch }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:columns-reject" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const events: SqlSessionChangeEvent[] = []; + session.onDidChange((event) => events.push(event)); + const pending = session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + await vi.advanceTimersByTimeAsync(0); + const result = await pending; + if (result.status !== "ready" || result.refreshToken === null) { + throw new Error("Expected retained column loading"); + } + + rejectBatch?.(new Error("offline")); + await Promise.resolve(); + await Promise.resolve(); + expect(events).toMatchObject([{ + reason: "catalog-availability", + refreshToken: result.refreshToken, + }]); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); + + it("expires timed-out column readiness without a stale event", async () => { + vi.useFakeTimers(); + try { + let resolveBatch: + | ((value: Awaited< + ReturnType + >) => void) + | undefined; + const batch = new Promise< + Awaited> + >((resolve) => { + resolveBatch = resolve; + }); + const service = createSqlLanguageService({ + columns: { id: "columns", loadColumns: () => batch }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:columns-expire" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const events: SqlSessionChangeEvent[] = []; + session.onDidChange((event) => events.push(event)); + const completion = session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + await vi.advanceTimersByTimeAsync(0); + await completion; + await vi.advanceTimersByTimeAsync(1_000); + + resolveBatch?.({ + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + }); + await Promise.resolve(); + await Promise.resolve(); + expect(events).toEqual([]); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); }); describe("namespace completion session integration", () => { @@ -1621,6 +1809,49 @@ describe("namespace completion session integration", () => { service.dispose(); }); + it("polls a provider-declared loading namespace with token correlation", async () => { + vi.useFakeTimers(); + try { + const service = serviceWithNamespaces(async () => ({ + epoch: { generation: 1, token: "epoch-1" }, + status: "loading", + })); + const text = "SELECT * FROM ma"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:namespaces-loading" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const events: SqlSessionChangeEvent[] = []; + session.onDidChange((event) => events.push(event)); + const result = await session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }); + if (result.status !== "ready" || result.refreshToken === null) { + throw new Error("Expected retained namespace loading"); + } + expect(result.value).toMatchObject({ + issues: [{ + reason: "namespace-catalog-loading", + remainingIntentLeaseMs: 1_000, + }], + }); + + await vi.advanceTimersByTimeAsync(100); + expect(events).toMatchObject([{ + reason: "catalog-availability", + refreshToken: result.refreshToken, + }]); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); + it("retains auxiliary owners across unrelated context changes", () => { const service = createSqlLanguageService({ columns: { @@ -1661,6 +1892,101 @@ describe("namespace completion session integration", () => { expect(session.revision).toBe(revision); service.dispose(); }); + + it("invalidates auxiliary caches with the relation catalog epoch", async () => { + let generation = 1; + let columnCalls = 0; + let namespaceCalls = 0; + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = createSqlLanguageService({ + catalog: { + id: "relations", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation, token: `epoch-${generation}` }, + relations: [], + status: "ready", + }), + subscribe: (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + }, + columns: { + id: "columns", + loadColumns: async (request) => { + columnCalls += 1; + return { + epoch: { generation, token: `epoch-${generation}` }, + relations: request.relations.map((relation) => ({ + columns: [], + coverage: "complete", + relationEntityId: relation.requestKey, + requestKey: relation.requestKey, + status: "ready", + })), + }; + }, + }, + dialects: [duckdb], + namespaces: { + id: "namespaces", + search: async () => { + namespaceCalls += 1; + return { + containers: [], + coverage: "complete", + epoch: { generation, token: `epoch-${generation}` }, + status: "ready", + }; + }, + }, + }); + const open = (text: string) => + service.openDocument({ + context: { + catalog: { scope: "connection:invalidate" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const columns = open("SELECT u. FROM users u"); + const namespaces = open("SELECT * FROM ma"); + const completeColumns = () => + columns.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + const completeNamespaces = () => + namespaces.complete({ + position: "SELECT * FROM ma".length, + trigger: { kind: "invoked" }, + }); + + await completeColumns(); + await completeNamespaces(); + await completeColumns(); + await completeNamespaces(); + expect({ columnCalls, namespaceCalls }).toEqual({ + columnCalls: 1, + namespaceCalls: 1, + }); + + generation = 2; + invalidate?.({ + epoch: { generation, token: `epoch-${generation}` }, + }); + await completeColumns(); + await completeNamespaces(); + expect({ columnCalls, namespaceCalls }).toEqual({ + columnCalls: 2, + namespaceCalls: 2, + }); + service.dispose(); + }); }); describe("statement-index session cache", () => { @@ -3979,10 +4305,10 @@ describe("session coverage hardening", () => { { namespaces: { id: "", search: async () => ({}) } }, ])("rejects a malformed auxiliary provider %#", (provider) => { expectSessionError("invalid-service-options", () => { - createSqlLanguageService({ + Reflect.apply(createSqlLanguageService, undefined, [{ ...provider, dialects: [duckdb], - }); + }]); }); }); @@ -4450,6 +4776,15 @@ describe("session coverage hardening", () => { }, completion: { catalogResponseBudgetMs: 0 }, dialects: [duckdb], + namespaces: { + id: "retained-intent-namespaces", + search: async () => ({ + containers: [], + coverage: "complete", + epoch: { generation: 0, token: "ready" }, + status: "ready", + }), + }, }); const session = service.openDocument({ context: { diff --git a/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts index dc306e5..8fd02df 100644 --- a/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts +++ b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts @@ -170,6 +170,8 @@ test("vNext editor applies a batched column completion", async () => { id: "browser-columns", loadColumns: async (request) => { columnCalls += 1; + const relation = request.relations[0]; + if (!relation) throw new Error("Expected one relation"); return { epoch: { generation: 1, token: "epoch-1" }, relations: [{ @@ -182,7 +184,7 @@ test("vNext editor applies a batched column completion", async () => { }], coverage: "complete", relationEntityId: "users", - requestKey: request.relations[0]?.requestKey, + requestKey: relation.requestKey, status: "ready", }], }; diff --git a/src/vnext/codemirror/sql-editor.ts b/src/vnext/codemirror/sql-editor.ts index 24f1cba..1696a65 100644 --- a/src/vnext/codemirror/sql-editor.ts +++ b/src/vnext/codemirror/sql-editor.ts @@ -181,7 +181,12 @@ function loadingLeaseMs( result: Extract, ): number | null { for (const issue of result.value.issues) { - if (issue.reason === "catalog-loading") { + if ( + (issue.reason === "catalog-loading" || + issue.reason === "column-catalog-loading" || + issue.reason === "namespace-catalog-loading") && + typeof issue.remainingIntentLeaseMs === "number" + ) { return issue.remainingIntentLeaseMs; } } diff --git a/src/vnext/column-catalog-batch-coordinator.ts b/src/vnext/column-catalog-batch-coordinator.ts index 0a4a378..bdaec68 100644 --- a/src/vnext/column-catalog-batch-coordinator.ts +++ b/src/vnext/column-catalog-batch-coordinator.ts @@ -185,7 +185,6 @@ function cacheKey( segment(request.dialectId), segment(String(epoch.generation)), segment(epoch.token), - segment(reference.relationEntityId ?? ""), segment(path), segment(searchPaths), ].join(""); diff --git a/src/vnext/column-catalog-boundary.ts b/src/vnext/column-catalog-boundary.ts index 179c00a..13a03fe 100644 --- a/src/vnext/column-catalog-boundary.ts +++ b/src/vnext/column-catalog-boundary.ts @@ -278,7 +278,7 @@ function decodeRelationReference( ): SqlColumnCatalogRelationReference | null { const record = readRecord( value, - new Set(["path", "relationEntityId", "requestKey"]), + new Set(["path", "requestKey"]), ); if (!record) return null; const requestKey = boundedString( @@ -289,15 +289,10 @@ function decodeRelationReference( required(record, "path"), MAX_COLUMN_RELATION_PATH_COMPONENTS, ); - const relationEntityId = optionalBoundedString( - required(record, "relationEntityId"), - MAX_COLUMN_ENTITY_ID_LENGTH, - ); - if (requestKey === null || !path || relationEntityId === null) return null; + if (requestKey === null || !path) return null; return Object.freeze({ path, requestKey, - ...(relationEntityId === undefined ? {} : { relationEntityId }), }); } @@ -659,12 +654,6 @@ export function decodeSqlColumnCatalogBatchResponse( const requested = new Set( request.relations.map((relation) => relation.requestKey), ); - const references = new Map( - request.relations.map((relation) => [ - relation.requestKey, - relation, - ]), - ); const byId = new Map(); let columnCount = 0; for (let index = 0; index < relationCount; index += 1) { @@ -681,14 +670,6 @@ export function decodeSqlColumnCatalogBatchResponse( if (!requested.has(relation.requestKey)) { return malformed("unexpected-relation"); } - const reference = references.get(relation.requestKey); - if ( - relation.status === "ready" && - reference?.relationEntityId !== undefined && - relation.relationEntityId !== reference.relationEntityId - ) { - return malformed("invalid-shape"); - } if (byId.has(relation.requestKey)) { return malformed("duplicate-request-key"); } diff --git a/src/vnext/column-catalog-types.ts b/src/vnext/column-catalog-types.ts index 8cf9e8c..f2a4690 100644 --- a/src/vnext/column-catalog-types.ts +++ b/src/vnext/column-catalog-types.ts @@ -8,7 +8,6 @@ export type SqlColumnCatalogCoverage = "complete" | "partial"; export interface SqlColumnCatalogRelationReference { readonly path: SqlIdentifierPath; - readonly relationEntityId?: string; readonly requestKey: string; } @@ -72,11 +71,30 @@ export interface SqlColumnCatalogBatchResponse { readonly relations: readonly SqlColumnCatalogRelationResult[]; } +export type SqlColumnCatalogProviderRelationResult = + | { + readonly columns: readonly SqlColumnCatalogColumn[]; + readonly coverage: SqlColumnCatalogCoverage; + readonly relationEntityId: string; + readonly requestKey: string; + readonly status: "ready"; + } + | Extract< + SqlColumnCatalogRelationResult, + { readonly status: "loading" | "failed" } + >; + +export interface SqlColumnCatalogProviderResponse { + readonly epoch: SqlCatalogEpoch; + readonly relations: + readonly SqlColumnCatalogProviderRelationResult[]; +} + export interface SqlColumnCatalogProvider { readonly id: string; readonly loadColumns: ( this: void, request: SqlColumnCatalogBatchRequest, signal: AbortSignal, - ) => Promise; + ) => Promise; } diff --git a/src/vnext/column-completion.ts b/src/vnext/column-completion.ts index b0b249e..228f8f2 100644 --- a/src/vnext/column-completion.ts +++ b/src/vnext/column-completion.ts @@ -11,6 +11,7 @@ import type { } from "./column-query-site.js"; import type { SqlColumnCatalogProviderReport, + SqlColumnCatalogFailure, SqlCompletionIssue, SqlCompletionItem, SqlCompletionList, @@ -187,13 +188,17 @@ function compareItems( left: SqlCompletionItem, right: SqlCompletionItem, ): number { - return left.label.localeCompare(right.label) || - (left.detail ?? "").localeCompare(right.detail ?? "") || - left.edit.insert.localeCompare(right.edit.insert) || + return compareText(left.label, right.label) || + compareText(left.detail ?? "", right.detail ?? "") || + compareText(left.edit.insert, right.edit.insert) || left.edit.from - right.edit.from || left.edit.to - right.edit.to; } +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + export function composeSqlColumnCompletion( input: { readonly dialect: SqlRelationDialectRuntime; @@ -224,6 +229,7 @@ export function composeSqlColumnCompletion( let hasFailure = false; let hasPartial = input.site.coverage === "partial"; const seen = new Set(); + const failures: SqlColumnCatalogFailure[] = []; for (const result of input.outcome.relations) { issues.push(...resultIssues(result)); if (result.status === "loading") { @@ -232,6 +238,11 @@ export function composeSqlColumnCompletion( } if (result.status === "failed") { hasFailure = true; + failures.push(Object.freeze({ + code: result.code, + requestKey: result.requestKey, + retry: result.retry, + })); continue; } if (result.coverage === "partial") hasPartial = true; @@ -292,21 +303,30 @@ export function composeSqlColumnCompletion( new Map(issues.map((item) => [item.reason, item])).values(), ); const coverage = hasPartial || hasFailure ? "partial" : "complete"; + const frozenFailures = Object.freeze(failures); + const firstFailure = failures[0]; const source: SqlColumnCatalogProviderReport = hasLoading ? { feature: "column-catalog", + failures: frozenFailures, outcome: "loading", providerId: input.outcome.providerId, } - : hasFailure && items.length === 0 + : hasFailure && items.length === 0 && + firstFailure !== undefined ? { feature: "column-catalog", + failures: Object.freeze([ + firstFailure, + ...failures.slice(1), + ]), outcome: "failed", providerId: input.outcome.providerId, } : { coverage, feature: "column-catalog", + failures: frozenFailures, outcome: "ready", providerId: input.outcome.providerId, }; diff --git a/src/vnext/column-query-site.ts b/src/vnext/column-query-site.ts index 9c5dc00..c867225 100644 --- a/src/vnext/column-query-site.ts +++ b/src/vnext/column-query-site.ts @@ -465,22 +465,32 @@ export function recognizeSqlColumnQuerySite( index += 1 ) { const token = tokens[index]!; + if ( + clause === "join-condition" && + token.from >= position + ) { + break; + } if (token.depth < selectDepth) break; if (token.depth !== selectDepth) { if (word(source, token) === "select") issues.add("nested-query"); continue; } const tokenWord = word(source, token); + if ( + tokenWord === "union" || + tokenWord === "intersect" || + tokenWord === "except" + ) { + break; + } if ( tokenWord === "where" || tokenWord === "group" || tokenWord === "having" || tokenWord === "qualify" || tokenWord === "order" || - tokenWord === "limit" || - tokenWord === "union" || - tokenWord === "intersect" || - tokenWord === "except" + tokenWord === "limit" ) { inFrom = false; commaStartsRelation = false; diff --git a/src/vnext/index.ts b/src/vnext/index.ts index 91de922..d5bc733 100644 --- a/src/vnext/index.ts +++ b/src/vnext/index.ts @@ -12,6 +12,8 @@ export type { SqlColumnCatalogCoverage, SqlColumnCatalogProvider, SqlColumnCatalogProvenance, + SqlColumnCatalogProviderRelationResult, + SqlColumnCatalogProviderResponse, SqlColumnCatalogRelationReference, SqlColumnCatalogRelationResult, SqlColumnCatalogResolvedColumn, @@ -20,6 +22,7 @@ export type { SqlCanonicalNamespacePath, SqlNamespaceCatalogContainer, SqlNamespaceCatalogProvider, + SqlNamespaceCatalogProviderResponse, SqlNamespaceCatalogProvenance, SqlNamespaceCatalogResolvedContainer, SqlNamespaceCatalogSearchRequest, @@ -77,6 +80,7 @@ export type { SqlCatalogSearchResponse, SqlCompletionCancellationReason, SqlColumnCatalogProviderReport, + SqlColumnCatalogFailure, SqlColumnCompletionProvenance, SqlCompletionProviderReport, SqlCompletionIssue, diff --git a/src/vnext/local-relation-site.ts b/src/vnext/local-relation-site.ts index 0c09230..27ad6cd 100644 --- a/src/vnext/local-relation-site.ts +++ b/src/vnext/local-relation-site.ts @@ -216,10 +216,33 @@ export function analyzeSqlLocalColumnSite( status: "unavailable", }); } - return recognizeSqlColumnQuerySite( + const result = recognizeSqlColumnQuerySite( context.source, context.slot, position, context.dialect, ); + if (result.status !== "ready") return result; + const visibility = visibleSqlCtesAt( + context.layout, + position - context.slot.source.from, + ); + const relations = result.relations.filter((relation) => { + const name = relation.path.length === 1 + ? relation.path[0] + : undefined; + return name === undefined || + !visibility.ctes.some((cte) => + context.dialect.completion.compareCteIdentifiers( + name, + cte.name, + ) === "equal" + ); + }); + return relations.length === result.relations.length + ? result + : Object.freeze({ + ...result, + relations: Object.freeze(relations), + }); } diff --git a/src/vnext/namespace-catalog-types.ts b/src/vnext/namespace-catalog-types.ts index 211a742..bcf62f1 100644 --- a/src/vnext/namespace-catalog-types.ts +++ b/src/vnext/namespace-catalog-types.ts @@ -68,6 +68,18 @@ export type SqlNamespaceCatalogSearchResponse = readonly status: "failed"; }; +export type SqlNamespaceCatalogProviderResponse = + | { + readonly containers: readonly SqlNamespaceCatalogContainer[]; + readonly coverage: "complete" | "partial"; + readonly epoch: SqlCatalogEpoch; + readonly status: "ready"; + } + | Extract< + SqlNamespaceCatalogSearchResponse, + { readonly status: "loading" | "failed" } + >; + export interface SqlNamespaceCatalogProvenance { readonly containerEntityId: string; readonly epoch: SqlCatalogEpoch; @@ -86,7 +98,7 @@ export interface SqlNamespaceCatalogProvider { this: void, request: SqlNamespaceCatalogSearchRequest, signal: AbortSignal, - ) => Promise; + ) => Promise; } export interface SqlNamespaceQuerySite { diff --git a/src/vnext/namespace-completion.ts b/src/vnext/namespace-completion.ts index 56f0ee0..d77215e 100644 --- a/src/vnext/namespace-completion.ts +++ b/src/vnext/namespace-completion.ts @@ -217,9 +217,11 @@ export function composeSqlNamespaceCompletion( if (response.status === "failed") { return Object.freeze({ source: Object.freeze({ + code: response.code, feature: "namespace-catalog", outcome: "failed", providerId: input.outcome.providerId, + retry: response.retry, }), value: list([], ["namespace-catalog-failed"]), }); diff --git a/src/vnext/relation-completion-types.ts b/src/vnext/relation-completion-types.ts index f5864d5..c6057fe 100644 --- a/src/vnext/relation-completion-types.ts +++ b/src/vnext/relation-completion-types.ts @@ -300,6 +300,12 @@ export type SqlCompletionIssue = readonly reason: "catalog-loading"; readonly remainingIntentLeaseMs: number; } + | { + readonly reason: + | "column-catalog-loading" + | "namespace-catalog-loading"; + readonly remainingIntentLeaseMs?: number; + } | { readonly reason: | "catalog-partial" @@ -310,12 +316,10 @@ export type SqlCompletionIssue = | "catalog-queue-timeout" | "catalog-timeout" | "column-catalog-failed" - | "column-catalog-loading" | "column-catalog-malformed" | "column-catalog-partial" | "cte-scope-uncertainty" | "namespace-catalog-failed" - | "namespace-catalog-loading" | "namespace-catalog-malformed" | "namespace-catalog-partial" | "namespace-prefix-uncertain" @@ -389,16 +393,22 @@ export type SqlColumnCatalogProviderReport = readonly outcome: "ready"; readonly providerId: string; readonly coverage: "complete" | "partial"; + readonly failures: readonly SqlColumnCatalogFailure[]; } | { readonly feature: "column-catalog"; readonly outcome: "loading"; readonly providerId: string; + readonly failures: readonly SqlColumnCatalogFailure[]; } | { readonly feature: "column-catalog"; readonly outcome: "failed"; readonly providerId: string; + readonly failures: readonly [ + SqlColumnCatalogFailure, + ...SqlColumnCatalogFailure[], + ]; } | { readonly feature: "column-catalog"; @@ -411,6 +421,12 @@ export type SqlColumnCatalogProviderReport = | "provider-failed"; }; +export interface SqlColumnCatalogFailure { + readonly code: SqlCatalogFailureCode; + readonly requestKey: string; + readonly retry: SqlCatalogRetryPolicy; +} + export type SqlNamespaceCatalogProviderReport = | { readonly coverage: "complete" | "partial"; @@ -427,6 +443,8 @@ export type SqlNamespaceCatalogProviderReport = readonly feature: "namespace-catalog"; readonly outcome: "failed"; readonly providerId: string; + readonly code: SqlCatalogFailureCode; + readonly retry: SqlCatalogRetryPolicy; } | { readonly feature: "namespace-catalog"; diff --git a/src/vnext/session.ts b/src/vnext/session.ts index cc16f32..c4f7971 100644 --- a/src/vnext/session.ts +++ b/src/vnext/session.ts @@ -124,6 +124,7 @@ const MAX_DIALECTS = 1_000; const DEFAULT_CATALOG_RESPONSE_BUDGET_MS = 40; const MAX_CATALOG_RESPONSE_BUDGET_MS = 50; const TERMINAL_LOADING_INTENT_LEASE_MS = 1_000; +const AUXILIARY_LOADING_RETRY_DELAY_MS = 100; interface ResolvedCatalogContext { readonly scope: string; @@ -133,11 +134,11 @@ interface ResolvedCatalogContext { interface CompletionRequestState { cancelReason: "caller" | "disposed" | "superseded" | null; readonly revision: SqlRevision; - ticket: + readonly tickets: Set< | SqlCatalogSearchWorkTicket | SqlColumnCatalogBatchTicket | SqlNamespaceCatalogSearchTicket - | null; + >; readonly token: SqlCompletionRefreshToken; } @@ -564,6 +565,13 @@ function completionCancellationReason( return request.cancelReason ?? "superseded"; } +function cancelCompletionTickets( + request: CompletionRequestState | null, +): void { + if (!request) return; + for (const ticket of request.tickets) ticket.cancel(); +} + function namespaceCompletionList( composition: SqlNamespaceCompletionComposition, ): SqlCompletionList { @@ -626,6 +634,31 @@ function mergeCompletionLists( }); } +function completionListWithLoadingLease( + value: SqlCompletionList, + reason: + | "column-catalog-loading" + | "namespace-catalog-loading", + remainingIntentLeaseMs: number, +): SqlCompletionList { + const issues = value.issues.map((issue) => + issue.reason === reason + ? Object.freeze({ reason, remainingIntentLeaseMs }) + : issue + ); + const first = issues[0]; + if (!first) return value; + const incompleteIssues: [ + SqlCompletionIssue, + ...SqlCompletionIssue[], + ] = [first, ...issues.slice(1)]; + return Object.freeze({ + isIncomplete: true, + issues: Object.freeze(incompleteIssues), + items: value.items, + }); +} + interface MissingDataProperty { readonly found: false; } @@ -1209,6 +1242,49 @@ export class DefaultSqlDocumentSession clearTimeout(timer.handle); } + #retainAuxiliaryRefresh( + active: CompletionRequestState, + readiness: Promise, + ): number { + this.#clearSoftRefreshIntentTimer(); + this.#refreshIntent = active; + const cell: SessionTimerCell = { + active: true, + handle: undefined, + }; + this.#softRefreshIntentTimer = cell; + const expire = setTimeout(() => { + cell.active = false; + this.#softRefreshIntentTimer = null; + this.#refreshIntent = null; + cancelCompletionTickets(active); + }, TERMINAL_LOADING_INTENT_LEASE_MS); + cell.handle = expire; + void readiness.then( + () => { + if ( + !cell.active || + this.#softRefreshIntentTimer !== cell + ) { + return; + } + const commit = this.#prepareServiceChange({ + expected: active, + reason: "catalog-availability", + }); + commit?.(); + }, + () => { + const commit = this.#prepareServiceChange({ + expected: active, + reason: "catalog-availability", + }); + commit?.(); + }, + ); + return TERMINAL_LOADING_INTENT_LEASE_MS; + } + #dispatchChange(event: SqlSessionChangeEvent): void { if ( this.#disposed || @@ -1301,8 +1377,20 @@ export class DefaultSqlDocumentSession } } return (): undefined => { - active?.ticket?.cancel(); - if (intent !== active) intent?.ticket?.cancel(); + cancelCompletionTickets(active); + if (intent !== active) cancelCompletionTickets(intent); + if (change.reason === "catalog") { + this.#columnOwner?.dispose(); + this.#columnOwner = null; + this.#columnOwnerDialect = null; + this.#columnOwnerScope = null; + this.#namespaceOwner?.dispose(); + this.#namespaceOwner = null; + this.#namespaceOwnerDialect = null; + this.#namespaceOwnerScope = null; + this.#replaceColumnOwner(); + this.#replaceNamespaceOwner(); + } this.#dispatchChange(event); return undefined; }; @@ -1559,9 +1647,9 @@ export class DefaultSqlDocumentSession previousIntent.cancelReason = "superseded"; } const cancelPrevious = (): void => { - previousActive?.ticket?.cancel(); + cancelCompletionTickets(previousActive); if (previousIntent !== previousActive) { - previousIntent?.ticket?.cancel(); + cancelCompletionTickets(previousIntent); } if (this.#refreshIntent === previousIntent) { this.#refreshIntent = null; @@ -1574,7 +1662,7 @@ export class DefaultSqlDocumentSession const active: CompletionRequestState = { cancelReason: null, revision: snapshot.revision, - ticket: null, + tickets: new Set(), token: refreshToken, }; this.#activeCompletion = active; @@ -1602,7 +1690,7 @@ export class DefaultSqlDocumentSession active.cancelReason === null ) { active.cancelReason = "caller"; - active.ticket?.cancel(); + cancelCompletionTickets(active); } }; const makeInvocationInert = (): void => { @@ -1752,7 +1840,7 @@ export class DefaultSqlDocumentSession relations: preparedRelations.references, searchPaths: catalog.searchPaths, }); - active.ticket = ticket; + active.tickets.add(ticket); let responseTimer: | ReturnType | undefined; @@ -1783,12 +1871,14 @@ export class DefaultSqlDocumentSession ); } if (raced.kind === "timeout") { - ticket.cancel(); + const remainingIntentLeaseMs = + this.#retainAuxiliaryRefresh(active, ticket.result); return Object.freeze({ - refreshToken: null, + refreshToken: active.token, revision: snapshot.revision, sources: Object.freeze([Object.freeze({ feature: "column-catalog" as const, + failures: Object.freeze([]), outcome: "loading" as const, providerId: columnCoordinator.providerId, })]), @@ -1797,6 +1887,7 @@ export class DefaultSqlDocumentSession isIncomplete: true, issues: Object.freeze([Object.freeze({ reason: "column-catalog-loading" as const, + remainingIntentLeaseMs, })] as const), items: Object.freeze([]), }), @@ -1815,12 +1906,31 @@ export class DefaultSqlDocumentSession completionCancellationReason(active), ); } + const columnLoading = + composition.sources[0]?.outcome === "loading"; + const remainingIntentLeaseMs = columnLoading + ? this.#retainAuxiliaryRefresh( + active, + new Promise((resolve) => { + setTimeout( + resolve, + AUXILIARY_LOADING_RETRY_DELAY_MS, + ); + }), + ) + : 0; return Object.freeze({ - refreshToken: null, + refreshToken: columnLoading ? active.token : null, revision: snapshot.revision, sources: composition.sources, status: "ready", - value: composition.value, + value: columnLoading + ? completionListWithLoadingLease( + composition.value, + "column-catalog-loading", + remainingIntentLeaseMs, + ) + : composition.value, }); } cancelPrevious(); @@ -1883,7 +1993,7 @@ export class DefaultSqlDocumentSession if (this.#refreshIntent === previousIntent) { this.#refreshIntent = null; } - active.ticket = ticket; + active.tickets.add(ticket); const elapsed = Math.max( 0, performance.now() - completionStartedAt, @@ -2060,6 +2170,7 @@ export class DefaultSqlDocumentSession let namespaceComposition: SqlNamespaceCompletionComposition | null = null; + let namespaceLoadingLeaseMs = 0; const namespaceOwner = this.#namespaceOwner; const namespaceCoordinator = this.#namespaceCoordinator; if (catalog && namespaceOwner && namespaceCoordinator) { @@ -2075,7 +2186,7 @@ export class DefaultSqlDocumentSession MAX_NAMESPACE_RESULTS, ), ); - active.ticket = ticket; + active.tickets.add(ticket); const elapsed = Math.max( 0, performance.now() - completionStartedAt, @@ -2114,7 +2225,8 @@ export class DefaultSqlDocumentSession ); } if (raced.kind === "timeout") { - ticket.cancel(); + namespaceLoadingLeaseMs = + this.#retainAuxiliaryRefresh(active, ticket.result); namespaceComposition = Object.freeze({ source: Object.freeze({ feature: "namespace-catalog", @@ -2147,6 +2259,18 @@ export class DefaultSqlDocumentSession completionCancellationReason(active), ); } + if (namespaceComposition.source.outcome === "loading") { + namespaceLoadingLeaseMs = + this.#retainAuxiliaryRefresh( + active, + new Promise((resolve) => { + setTimeout( + resolve, + AUXILIARY_LOADING_RETRY_DELAY_MS, + ); + }), + ); + } } } @@ -2175,7 +2299,13 @@ export class DefaultSqlDocumentSession const value = namespaceComposition ? mergeCompletionLists( composition.value, - namespaceCompletionList(namespaceComposition), + namespaceLoadingLeaseMs > 0 + ? completionListWithLoadingLease( + namespaceCompletionList(namespaceComposition), + "namespace-catalog-loading", + namespaceLoadingLeaseMs, + ) + : namespaceCompletionList(namespaceComposition), ) : composition.value; const sources = namespaceComposition @@ -2186,7 +2316,8 @@ export class DefaultSqlDocumentSession : composition.sources; return Object.freeze({ refreshToken: - (catalogOutcome?.status === "loading" || + (namespaceLoadingLeaseMs > 0 || + catalogOutcome?.status === "loading" || (catalogOutcome?.status === "usable" && catalogOutcome.response.status === "loading")) && (this.#refreshIntent === active || @@ -2509,9 +2640,9 @@ export class DefaultSqlDocumentSession ) { refreshIntent.cancelReason = "superseded"; } - activeCompletion?.ticket?.cancel(); + cancelCompletionTickets(activeCompletion); if (refreshIntent !== activeCompletion) { - refreshIntent?.ticket?.cancel(); + cancelCompletionTickets(refreshIntent); } this.#replaceCatalogOwner(); this.#replaceColumnOwner(); @@ -2556,9 +2687,9 @@ export class DefaultSqlDocumentSession ) { refreshIntent.cancelReason = "disposed"; } - activeCompletion?.ticket?.cancel(); + cancelCompletionTickets(activeCompletion); if (refreshIntent !== activeCompletion) { - refreshIntent?.ticket?.cancel(); + cancelCompletionTickets(refreshIntent); } catalogOwner?.dispose(); columnOwner?.dispose(); diff --git a/test/vnext-types/marimo-sql-migration.test-d.ts b/test/vnext-types/marimo-sql-migration.test-d.ts index 8af1a77..deebda8 100644 --- a/test/vnext-types/marimo-sql-migration.test-d.ts +++ b/test/vnext-types/marimo-sql-migration.test-d.ts @@ -536,7 +536,6 @@ const coldColumnBatch = { { quoted: false, value: "main" }, { quoted: false, value: "users" }, ], - relationEntityId: "connection:users", requestKey: "binding:users", }, { @@ -544,7 +543,6 @@ const coldColumnBatch = { { quoted: false, value: "main" }, { quoted: false, value: "orders" }, ], - relationEntityId: "connection:orders", requestKey: "binding:orders", }, ], From 2b66021805d965389495627fef67f6207633aba5 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 03:36:15 +0800 Subject: [PATCH 20/25] fix(vnext): bound auxiliary catalog refresh --- docs/vnext/column-catalog-authority.md | 19 +- docs/vnext/namespace-catalog-authority.md | 7 + src/vnext/__tests__/session.test.ts | 179 +++++++++++++++++- .../codemirror/__tests__/sql-editor.test.ts | 4 + src/vnext/session.ts | 97 ++++++++-- src/vnext/types.ts | 2 + 6 files changed, 285 insertions(+), 23 deletions(-) diff --git a/docs/vnext/column-catalog-authority.md b/docs/vnext/column-catalog-authority.md index 5216f7a..a72aee4 100644 --- a/docs/vnext/column-catalog-authority.md +++ b/docs/vnext/column-catalog-authority.md @@ -5,8 +5,8 @@ Status: internal vertical-slice contract Column discovery is lazy, provider-owned, and batched. A completion request sends every unresolved relation reference in one provider invocation. Each reference carries a caller-local `requestKey`, a decoded identifier path, and -an optional previously authenticated relation entity ID. The provider resolves -paths against the supplied catalog scope, search paths, and dialect. +no unauthenticated entity identity. The provider resolves paths against the +supplied catalog scope, search paths, and dialect. The provider returns stable relation and column entity IDs. Every accepted column contains: @@ -37,11 +37,11 @@ entries from another epoch. The cache is LRU-bounded by relation entries. Only complete ready results are cached. Partial, loading, and failed results remain visible to the caller but are eligible for another provider request. -The initial coordinator does not subscribe to catalog invalidations. Until it -is connected to the relation catalog's private epoch coordinator, a host must -supersede or dispose the column owner when catalog authority changes. Session -integration must not treat a cached complete result as current across a known -catalog revision. +Relation-catalog subscription events invalidate the session's relation, column, +and namespace observations together. Hosts without a subscribed relation +provider call `session.invalidateCatalog()` when schema authority changes. The +method advances the session revision, cancels retained completion work, rebuilds +all catalog owners, and emits one `catalog` change event. ## Lifecycle @@ -55,3 +55,8 @@ cannot publish or populate the cache. The coordinator batches a request once, never once per relation. Cache hits and misses are composed deterministically while only misses are sent to the provider. + +Provider-declared loading receives at most one automatic retry for the same +document, context, and completion position. A persistent loading response is +then returned without a refresh token; hosts resume it through catalog +invalidation instead of an unbounded polling loop. diff --git a/docs/vnext/namespace-catalog-authority.md b/docs/vnext/namespace-catalog-authority.md index b71c098..500f81e 100644 --- a/docs/vnext/namespace-catalog-authority.md +++ b/docs/vnext/namespace-catalog-authority.md @@ -36,3 +36,10 @@ disposes it when catalog authority changes. Query-site integration calls passes the outcome plus the site's replacement range and dialect prefix matcher to `composeSqlNamespaceCompletion`. Namespace items are merged with local and relation-catalog items under the same bounded completion response budget. +Relation-catalog subscription events invalidate namespace observations. +Namespace-only hosts, and hosts whose relation provider has no subscription, +call `session.invalidateCatalog()` after a schema-authority change. + +Provider-declared loading receives at most one automatic retry for a stable +document, context, and completion position. Further loading responses do not +schedule polling; a host-provided catalog invalidation resumes discovery. diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index a7a6653..fd19472 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -1463,6 +1463,64 @@ describe("column completion session integration", () => { } }); + it("bounds provider-declared column loading to one automatic retry", async () => { + vi.useFakeTimers(); + try { + let calls = 0; + const service = serviceWithColumns(async (request) => { + calls += 1; + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: request.relations.map((relation) => ({ + requestKey: relation.requestKey, + status: "loading", + })), + }; + }); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:columns-loading" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const events: SqlSessionChangeEvent[] = []; + session.onDidChange((event) => events.push(event)); + const result = await session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + if (result.status !== "ready" || result.refreshToken === null) { + throw new Error("Expected retained column loading"); + } + + await vi.advanceTimersByTimeAsync(100); + expect(events).toMatchObject([{ + reason: "catalog-availability", + refreshToken: result.refreshToken, + }]); + const retry = await session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + expect(retry).toMatchObject({ + refreshToken: null, + sources: [{ + feature: "column-catalog", + outcome: "loading", + }], + }); + await vi.advanceTimersByTimeAsync(500); + expect(calls).toBe(2); + expect(events).toHaveLength(1); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); + it("emits readiness when a timed-out column batch settles", async () => { vi.useFakeTimers(); try { @@ -1812,10 +1870,14 @@ describe("namespace completion session integration", () => { it("polls a provider-declared loading namespace with token correlation", async () => { vi.useFakeTimers(); try { - const service = serviceWithNamespaces(async () => ({ - epoch: { generation: 1, token: "epoch-1" }, - status: "loading", - })); + let calls = 0; + const service = serviceWithNamespaces(async () => { + calls += 1; + return { + epoch: { generation: 1, token: "epoch-1" }, + status: "loading", + }; + }); const text = "SELECT * FROM ma"; const session = service.openDocument({ context: { @@ -1846,6 +1908,24 @@ describe("namespace completion session integration", () => { reason: "catalog-availability", refreshToken: result.refreshToken, }]); + + const retry = await session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }); + expect(retry).toMatchObject({ + refreshToken: null, + sources: [ + { feature: "relation-catalog" }, + { + feature: "namespace-catalog", + outcome: "loading", + }, + ], + }); + await vi.advanceTimersByTimeAsync(500); + expect(calls).toBe(2); + expect(events).toHaveLength(1); service.dispose(); } finally { vi.useRealTimers(); @@ -1987,6 +2067,97 @@ describe("namespace completion session integration", () => { }); service.dispose(); }); + + it("manually invalidates auxiliary-only catalog authority", async () => { + let generation = 1; + let columnCalls = 0; + let namespaceCalls = 0; + const service = createSqlLanguageService({ + columns: { + id: "columns", + loadColumns: async (request) => { + columnCalls += 1; + return { + epoch: { generation, token: `epoch-${generation}` }, + relations: request.relations.map((relation) => ({ + columns: [], + coverage: "complete", + relationEntityId: relation.requestKey, + requestKey: relation.requestKey, + status: "ready", + })), + }; + }, + }, + dialects: [duckdb], + namespaces: { + id: "namespaces", + search: async () => { + namespaceCalls += 1; + return { + containers: [], + coverage: "complete", + epoch: { generation, token: `epoch-${generation}` }, + status: "ready", + }; + }, + }, + }); + const columns = service.openDocument({ + context: { + catalog: { scope: "connection:manual-invalidate" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT u. FROM users u", + }); + const namespaces = service.openDocument({ + context: { + catalog: { scope: "connection:manual-invalidate" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ma", + }); + const completeColumns = () => + columns.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + const completeNamespaces = () => + namespaces.complete({ + position: "SELECT * FROM ma".length, + trigger: { kind: "invoked" }, + }); + + await completeColumns(); + await completeNamespaces(); + await completeColumns(); + await completeNamespaces(); + expect({ columnCalls, namespaceCalls }).toEqual({ + columnCalls: 1, + namespaceCalls: 1, + }); + + generation = 2; + const columnRevision = columns.invalidateCatalog(); + const namespaceRevision = namespaces.invalidateCatalog(); + expect(columns.revision).toBe(columnRevision); + expect(namespaces.revision).toBe(namespaceRevision); + await completeColumns(); + await completeNamespaces(); + expect({ columnCalls, namespaceCalls }).toEqual({ + columnCalls: 2, + namespaceCalls: 2, + }); + + columns.dispose(); + expect(() => columns.invalidateCatalog()).toThrow( + "SQL document session is disposed", + ); + namespaces.dispose(); + service.dispose(); + }); }); describe("statement-index session cache", () => { diff --git a/src/vnext/codemirror/__tests__/sql-editor.test.ts b/src/vnext/codemirror/__tests__/sql-editor.test.ts index 220b648..fe42c99 100644 --- a/src/vnext/codemirror/__tests__/sql-editor.test.ts +++ b/src/vnext/codemirror/__tests__/sql-editor.test.ts @@ -161,6 +161,10 @@ function fakeService( disposed = true; sessionDisposalCount += 1; }, + invalidateCatalog: () => { + revision = createSqlRevisionToken(); + return revision; + }, isCurrent: (candidate) => !disposed && candidate === revision, onDidChange: (nextListener) => { listener = nextListener; diff --git a/src/vnext/session.ts b/src/vnext/session.ts index c4f7971..05012e1 100644 --- a/src/vnext/session.ts +++ b/src/vnext/session.ts @@ -166,6 +166,12 @@ interface TerminalRefreshIntent { readonly token: SqlCompletionRefreshToken; } +interface AuxiliaryLoadingRetry { + readonly context: Context; + readonly position: number; + readonly source: SqlSourceSnapshot; +} + interface CompletionConfiguration { readonly catalogResponseBudgetMs: number; } @@ -1071,6 +1077,7 @@ export class DefaultSqlDocumentSession readonly #onDispose: () => void; readonly #listeners = new Set(); #activeCompletion: CompletionRequestState | null = null; + #columnLoadingRetry: AuxiliaryLoadingRetry | null = null; #catalogOwner: SqlCatalogSearchWorkOwner | null = null; #catalogOwnerDialect: SqlRelationDialectRuntime | null = null; #catalogOwnerScope: string | null = null; @@ -1080,6 +1087,7 @@ export class DefaultSqlDocumentSession #namespaceOwner: SqlNamespaceCatalogOwner | null = null; #namespaceOwnerDialect: SqlRelationDialectRuntime | null = null; #namespaceOwnerScope: string | null = null; + #namespaceLoadingRetry: AuxiliaryLoadingRetry | null = null; #disposed = false; #localRelationStatementCache: | LocalRelationStatementCache @@ -1285,6 +1293,34 @@ export class DefaultSqlDocumentSession return TERMINAL_LOADING_INTENT_LEASE_MS; } + #claimAuxiliaryLoadingRetry( + feature: "column" | "namespace", + snapshot: SessionSnapshot, + position: number, + ): boolean { + const previous = feature === "column" + ? this.#columnLoadingRetry + : this.#namespaceLoadingRetry; + if ( + previous?.context === snapshot.context && + previous.source === snapshot.source && + previous.position === position + ) { + return false; + } + const next = Object.freeze({ + context: snapshot.context, + position, + source: snapshot.source, + }); + if (feature === "column") { + this.#columnLoadingRetry = next; + } else { + this.#namespaceLoadingRetry = next; + } + return true; + } + #dispatchChange(event: SqlSessionChangeEvent): void { if ( this.#disposed || @@ -1380,6 +1416,8 @@ export class DefaultSqlDocumentSession cancelCompletionTickets(active); if (intent !== active) cancelCompletionTickets(intent); if (change.reason === "catalog") { + this.#columnLoadingRetry = null; + this.#namespaceLoadingRetry = null; this.#columnOwner?.dispose(); this.#columnOwner = null; this.#columnOwnerDialect = null; @@ -1522,6 +1560,25 @@ export class DefaultSqlDocumentSession }); }; + readonly invalidateCatalog = (): SqlRevision => { + if (this.#disposed) { + throw new SqlSessionError( + "session-disposed", + "SQL document session is disposed", + ); + } + const commit = this.#prepareServiceChange({ reason: "catalog" }); + if (!commit) { + throw new SqlSessionError( + "session-disposed", + "SQL document session is disposed", + ); + } + const revision = this.#snapshot.revision; + commit(); + return revision; + }; + readonly complete = ( request: SqlCompletionRequest, ): SqlCompletionTask => { @@ -1908,7 +1965,14 @@ export class DefaultSqlDocumentSession } const columnLoading = composition.sources[0]?.outcome === "loading"; - const remainingIntentLeaseMs = columnLoading + const retryLoading = + columnLoading && + this.#claimAuxiliaryLoadingRetry( + "column", + snapshot, + request.position, + ); + const remainingIntentLeaseMs = retryLoading ? this.#retainAuxiliaryRefresh( active, new Promise((resolve) => { @@ -1920,11 +1984,11 @@ export class DefaultSqlDocumentSession ) : 0; return Object.freeze({ - refreshToken: columnLoading ? active.token : null, + refreshToken: retryLoading ? active.token : null, revision: snapshot.revision, sources: composition.sources, status: "ready", - value: columnLoading + value: retryLoading ? completionListWithLoadingLease( composition.value, "column-catalog-loading", @@ -2261,15 +2325,21 @@ export class DefaultSqlDocumentSession } if (namespaceComposition.source.outcome === "loading") { namespaceLoadingLeaseMs = - this.#retainAuxiliaryRefresh( - active, - new Promise((resolve) => { - setTimeout( - resolve, - AUXILIARY_LOADING_RETRY_DELAY_MS, - ); - }), - ); + this.#claimAuxiliaryLoadingRetry( + "namespace", + snapshot, + request.position, + ) + ? this.#retainAuxiliaryRefresh( + active, + new Promise((resolve) => { + setTimeout( + resolve, + AUXILIARY_LOADING_RETRY_DELAY_MS, + ); + }), + ) + : 0; } } } @@ -2624,6 +2694,7 @@ export class DefaultSqlDocumentSession this.#localRelationStatementCache = null; } this.#activeCompletion = null; + this.#columnLoadingRetry = null; this.#refreshIntent = null; this.#clearSoftRefreshIntentTimer(); this.#clearTerminalIntent(); @@ -2667,8 +2738,10 @@ export class DefaultSqlDocumentSession this.#activeCompletion = null; this.#refreshIntent = null; this.#catalogOwner = null; + this.#columnLoadingRetry = null; this.#columnOwner = null; this.#namespaceOwner = null; + this.#namespaceLoadingRetry = null; this.#clearSoftRefreshIntentTimer(); this.#clearTerminalIntent(); this.#listeners.clear(); diff --git a/src/vnext/types.ts b/src/vnext/types.ts index 18a3089..0a081ba 100644 --- a/src/vnext/types.ts +++ b/src/vnext/types.ts @@ -135,6 +135,8 @@ export interface OpenSqlDocument { /** Owns all mutable state for one open SQL document. */ export interface SqlDocumentSession { readonly revision: SqlRevision; + /** Invalidates relation, column, and namespace catalog observations. */ + readonly invalidateCatalog: () => SqlRevision; readonly statementBoundaryAt: ( request: SqlStatementBoundaryAtRequest, ) => SqlStatementBoundaryAtResult; From 2d5b40242ad44f3c86de2bf4cd1baf7dd03af050 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 03:41:32 +0800 Subject: [PATCH 21/25] fix(vnext): expose bounded catalog invalidation --- docs/vnext/codemirror-adapter.md | 11 ++++++++ .../codemirror/__tests__/sql-editor.test.ts | 27 ++++++++++++++++++- src/vnext/codemirror/sql-editor.ts | 12 +++++++++ src/vnext/session.ts | 1 + 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/vnext/codemirror-adapter.md b/docs/vnext/codemirror-adapter.md index 8ab069d..d721701 100644 --- a/docs/vnext/codemirror-adapter.md +++ b/docs/vnext/codemirror-adapter.md @@ -87,6 +87,17 @@ const visible = support.statementBoundariesIntersecting(view, { ``` Both methods return `null` for foreign, destroyed, or unsynchronized views. +Catalog-backed hosts can invalidate the adapter-owned session without exposing +it: + +```ts +const revision = support.invalidateCatalog(view); +``` + +This returns `null` for foreign or destroyed views. A live view advances its +revision, cancels stale catalog work, and forces relation, column, and namespace +providers to be consulted again. + An opt-in structural gutter marks only lines intersecting scanner-owned SQL `code` spans. It never parses or copies statement text: diff --git a/src/vnext/codemirror/__tests__/sql-editor.test.ts b/src/vnext/codemirror/__tests__/sql-editor.test.ts index fe42c99..4b32cdd 100644 --- a/src/vnext/codemirror/__tests__/sql-editor.test.ts +++ b/src/vnext/codemirror/__tests__/sql-editor.test.ts @@ -48,6 +48,7 @@ interface FakeServiceHarness { readonly completeSignals: AbortSignal[]; readonly emit: (event: SqlSessionChangeEvent) => void; readonly getLastToken: () => SqlCompletionRefreshToken | null; + readonly invalidations: () => number; readonly service: SqlLanguageService; readonly sessionDisposals: () => number; readonly statementBoundaryCalls: () => number; @@ -138,6 +139,7 @@ function fakeService( | ((event: SqlSessionChangeEvent) => void) | null = null; let lastToken: SqlCompletionRefreshToken | null = null; + let invalidationCount = 0; let sessionDisposalCount = 0; let statementBoundaryCallCount = 0; let statementIntersectionCallCount = 0; @@ -162,6 +164,7 @@ function fakeService( sessionDisposalCount += 1; }, invalidateCatalog: () => { + invalidationCount += 1; revision = createSqlRevisionToken(); return revision; }, @@ -243,6 +246,7 @@ function fakeService( completeSignals, emit: (event) => listener?.(event), getLastToken: () => lastToken, + invalidations: () => invalidationCount, service, sessionDisposals: () => sessionDisposalCount, statementBoundaryCalls: () => statementBoundaryCallCount, @@ -360,7 +364,7 @@ async function resolveCompletionInfo( } describe("sqlEditor", () => { - it("exposes current statement boundaries only for owned views", () => { + it("exposes session controls only for owned views", () => { const service = createSqlLanguageService({ dialects: [duckdbDialect()], }); @@ -383,6 +387,8 @@ describe("sqlEditor", () => { from: 0, to: view.state.doc.length, })?.boundaries).toHaveLength(2); + expect(support.invalidateCatalog(view)).not.toBeNull(); + expect(support.invalidateCatalog(foreign)).toBeNull(); expect(support.statementBoundaryAt(foreign, { affinity: "left", position: 0, @@ -397,6 +403,7 @@ describe("sqlEditor", () => { })).toBeNull(); view.destroy(); + expect(support.invalidateCatalog(view)).toBeNull(); expect(support.statementBoundaryAt(view, { affinity: "left", position: 0, @@ -408,6 +415,24 @@ describe("sqlEditor", () => { service.dispose(); }); + it("proxies catalog invalidation only to its owned live session", () => { + const harness = fakeService((revision) => readyResult(revision, [])); + const support = sqlEditor({ + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + const foreign = createView([]); + + expect(support.invalidateCatalog(view)).not.toBeNull(); + expect(harness.invalidations()).toBe(1); + expect(support.invalidateCatalog(foreign)).toBeNull(); + expect(harness.invalidations()).toBe(1); + view.destroy(); + expect(support.invalidateCatalog(view)).toBeNull(); + expect(harness.invalidations()).toBe(1); + }); + it("renders an opt-in visible-line statement gutter", async () => { const service = createSqlLanguageService({ dialects: [duckdbDialect()], diff --git a/src/vnext/codemirror/sql-editor.ts b/src/vnext/codemirror/sql-editor.ts index 1696a65..efd8366 100644 --- a/src/vnext/codemirror/sql-editor.ts +++ b/src/vnext/codemirror/sql-editor.ts @@ -95,6 +95,7 @@ export interface SqlEditorSupport< readonly SqlEmbeddedRegion[] >; readonly extension: Extension; + readonly invalidateCatalog: (view: EditorView) => SqlRevision | null; readonly statementBoundariesIntersecting: ( view: EditorView, request: SqlStatementBoundariesIntersectingRequest, @@ -671,6 +672,15 @@ export function createSqlEditorInternal< this.#clearCompletionState(); }; + readonly invalidateCatalog = (): SqlRevision | null => { + if (this.#destroyed) return null; + try { + return this.#session.invalidateCatalog(); + } catch { + return null; + } + }; + readonly statementBoundariesIntersecting = ( request: SqlStatementBoundariesIntersectingRequest, ): SqlStatementBoundariesIntersectingResult | null => { @@ -869,6 +879,8 @@ export function createSqlEditorInternal< override: [completionSource, ...externalSources], }), ], + invalidateCatalog: (view: EditorView): SqlRevision | null => + view.plugin(plugin)?.invalidateCatalog() ?? null, statementBoundariesIntersecting: ( view: EditorView, request: SqlStatementBoundariesIntersectingRequest, diff --git a/src/vnext/session.ts b/src/vnext/session.ts index 05012e1..bf15af8 100644 --- a/src/vnext/session.ts +++ b/src/vnext/session.ts @@ -2695,6 +2695,7 @@ export class DefaultSqlDocumentSession } this.#activeCompletion = null; this.#columnLoadingRetry = null; + this.#namespaceLoadingRetry = null; this.#refreshIntent = null; this.#clearSoftRefreshIntentTimer(); this.#clearTerminalIntent(); From b139e7be4c2c8e674b5efad5255f87c8b884a737 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 12:32:46 +0800 Subject: [PATCH 22/25] fix(vnext): harden completion review boundaries --- docs/vnext/column-catalog-authority.md | 11 +- docs/vnext/marimo-sql-migration.md | 27 +- .../column-catalog-batch-coordinator.test.ts | 111 +++++ .../__tests__/column-catalog-boundary.test.ts | 114 +++++ src/vnext/__tests__/column-completion.test.ts | 48 ++ src/vnext/__tests__/column-query-site.test.ts | 121 +++++- src/vnext/__tests__/hostile-data.test.ts | 15 + .../__tests__/local-relation-site.test.ts | 28 ++ .../namespace-catalog-boundary.test.ts | 90 +++- .../namespace-catalog-coordinator.bench.ts | 6 +- .../namespace-catalog-coordinator.test.ts | 77 ++++ .../node-sql-parser-query-bindings.test.ts | 411 +++++++++++++++++- .../__tests__/query-binding-model.bench.ts | 68 +++ .../__tests__/query-binding-model.test.ts | 114 +++++ src/vnext/__tests__/session.test.ts | 94 ++++ .../codemirror/__tests__/sql-editor.test.ts | 97 +++++ .../sql-editor-completion-gate.test.ts | 98 ++++- .../browser_tests/statement-gutter.test.ts | 16 +- src/vnext/codemirror/sql-editor.ts | 69 ++- src/vnext/column-catalog-batch-coordinator.ts | 173 ++++---- src/vnext/column-catalog-boundary.ts | 37 +- src/vnext/column-completion.ts | 18 +- src/vnext/column-query-site.ts | 243 ++++++++--- src/vnext/namespace-catalog-boundary.ts | 45 +- src/vnext/namespace-catalog-coordinator.ts | 177 ++++---- src/vnext/node-sql-parser-query-bindings.ts | 294 ++++++++++--- src/vnext/query-binding-model.ts | 117 +++-- src/vnext/session.ts | 169 +++---- src/vnext/types.ts | 10 + 29 files changed, 2373 insertions(+), 525 deletions(-) create mode 100644 src/vnext/__tests__/hostile-data.test.ts diff --git a/docs/vnext/column-catalog-authority.md b/docs/vnext/column-catalog-authority.md index a72aee4..aff30c0 100644 --- a/docs/vnext/column-catalog-authority.md +++ b/docs/vnext/column-catalog-authority.md @@ -3,10 +3,13 @@ Status: internal vertical-slice contract Column discovery is lazy, provider-owned, and batched. A completion request -sends every unresolved relation reference in one provider invocation. Each -reference carries a caller-local `requestKey`, a decoded identifier path, and -no unauthenticated entity identity. The provider resolves paths against the -supplied catalog scope, search paths, and dialect. +sends at most 64 unresolved relation references in one provider invocation. +The deterministic first batch is useful but explicitly partial when more +visible relations match; completion reports `query-binding-partial` and never +claims the omitted relations were searched. Each reference carries a +caller-local `requestKey`, a decoded identifier path, and no unauthenticated +entity identity. The provider resolves paths against the supplied catalog +scope, search paths, and dialect. The provider returns stable relation and column entity IDs. Every accepted column contains: diff --git a/docs/vnext/marimo-sql-migration.md b/docs/vnext/marimo-sql-migration.md index 66b3a95..77d17cc 100644 --- a/docs/vnext/marimo-sql-migration.md +++ b/docs/vnext/marimo-sql-migration.md @@ -42,6 +42,11 @@ The column authority resolves each canonical relation path within the same scope and search paths as relation completion. Column IDs and returned relation IDs are stable within that connection incarnation. +The library sends at most 64 matching physical relations in one request. +Larger visible sets are deterministically truncated and reported as partial; +the provider must not interpret one batch as exhaustive when completion carries +`query-binding-partial`. + Each column supplies a canonical identifier and provider-rendered `insertText`; these are intentionally separate so quoted or dialect-sensitive names insert correctly. The provider also supplies ordinal, data type, and @@ -128,7 +133,12 @@ stable provenance, cancellation, and batched column work. The remaining feature gaps are: -- the dialect coverage described above. +- the dialect coverage described above; and +- output-column inference for CTEs and derived relations. vNext completes CTE + relation names but deliberately does not send a visible CTE name to the + physical column provider. Keep the legacy column source for those sites, or + defer full source replacement, until projection/output-column inference + lands. The fixture feeds marimo's immutable namespace projection—stable entity ID, scope, canonical identifier path, and namespace kind—through one public, @@ -146,8 +156,8 @@ scoped namespace provider on the shared service. 4. Compare relation and column results with the golden corpus, including quoted insert text, aliases, ambiguity, partial/loading/failure states, and cold epoch behavior. -5. Cut over the four supported dialects as one source replacement, preserving - variable and keyword external sources. +5. Cut over supported physical-relation sites while preserving variable, + keyword, and legacy CTE/derived-output column sources. 6. Add dialect coverage, expand the router, and remove completion-only legacy schema code. Keep legacy schema data while hover or diagnostics still use it. @@ -169,10 +179,13 @@ The compile-only marimo fixture proves: - preservation of variable and keyword sources; and - supported-dialect vNext routing with exclusive legacy fallback. -Library tests cover relation and column completion for `FROM`, `JOIN`, -`alias.`, unqualified projections and predicates, `USING`, nested queries, -CTEs, quoted identifiers, ambiguous columns, provider loading/invalidations, -bounded batching, and template barriers. +Library tests cover relation and physical-column completion for `FROM`, `JOIN`, +`alias.`, unqualified projections and predicates, `USING`, correlated nested +queries, quoted identifiers, ambiguous columns, provider +loading/invalidations, bounded batching, and template barriers. CTE tests prove +declaration-order relation visibility and that CTE names are not incorrectly +resolved through the physical column provider; they do not prove CTE +output-column inference. Marimo integration tests cover: diff --git a/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts index 8f2c8c2..97fb1c4 100644 --- a/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts +++ b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts @@ -327,6 +327,89 @@ describe("column catalog batch coordinator", () => { other.cancel(); }); + it("preserves the newest request across reentrant abort handlers", async () => { + const signals: AbortSignal[] = []; + let owner: ReturnType["owner"] | null = null; + const reentrant: { + ticket: + | ReturnType["owner"]["request"]> + | null; + } = { ticket: null }; + const configured = setup((_request, signal) => { + signals.push(signal); + if (signals.length === 1) { + signal.addEventListener("abort", () => { + reentrant.ticket = + owner?.request(input([reference("reentrant")])) ?? null; + }, { once: true }); + } + return new Promise(() => undefined); + }); + owner = configured.owner; + const first = owner.request(input([reference("first")])); + const interrupted = owner.request(input([reference("interrupted")])); + + await expect(first.result).resolves.toEqual({ + status: "superseded", + }); + await expect(interrupted.result).resolves.toEqual({ + status: "superseded", + }); + expect(signals).toHaveLength(2); + expect(signals.map((signal) => signal.aborted)).toEqual([ + true, + false, + ]); + + const newest = owner.request(input([reference("newest")])); + expect(signals.map((signal) => signal.aborted)).toEqual([ + true, + true, + false, + ]); + await expect(reentrant.ticket?.result).resolves.toEqual({ + status: "superseded", + }); + newest.cancel(); + }); + + it.each(["owner", "coordinator"] as const)( + "does not resurrect work after reentrant %s disposal", + async (target) => { + const signals: AbortSignal[] = []; + let dispose = (): void => {}; + const configured = setup((_request, signal) => { + signals.push(signal); + if (signals.length === 1) { + signal.addEventListener("abort", () => dispose(), { + once: true, + }); + } + return new Promise(() => undefined); + }); + dispose = target === "owner" + ? configured.owner.dispose + : configured.coordinator.dispose; + const first = configured.owner.request( + input([reference("first")]), + ); + const interrupted = configured.owner.request( + input([reference("interrupted")]), + ); + + await expect(first.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + await expect(interrupted.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + expect(signals).toHaveLength(1); + expect(signals[0]?.aborted).toBe(true); + }, + ); + it("disposes owners and coordinator with prompt aborts", async () => { const signals: AbortSignal[] = []; const { coordinator, owner } = setup((_request, signal) => { @@ -386,6 +469,34 @@ describe("column catalog batch coordinator", () => { }); }); + it("settles a revoked-array provider response as malformed", async () => { + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + const { owner } = setup(() => ({ + epoch, + relations: revoked.proxy, + })); + await expect(owner.request(input()).result).resolves.toEqual({ + reason: "malformed-response", + status: "unavailable", + }); + }); + + it("contains an unexpected decoder exception", async () => { + const integerCheck = vi.spyOn(Number, "isSafeInteger"); + const { owner } = setup((request) => { + integerCheck.mockImplementationOnce(() => { + throw new Error("decoder"); + }); + return ready(request); + }); + await expect(owner.request(input()).result).resolves.toEqual({ + reason: "malformed-response", + status: "unavailable", + }); + integerCheck.mockRestore(); + }); + it("bounds cache entries with deterministic LRU eviction", async () => { const calls = new Map(); const { owner } = setup((request) => { diff --git a/src/vnext/__tests__/column-catalog-boundary.test.ts b/src/vnext/__tests__/column-catalog-boundary.test.ts index 06a8f8b..2ab1f9e 100644 --- a/src/vnext/__tests__/column-catalog-boundary.test.ts +++ b/src/vnext/__tests__/column-catalog-boundary.test.ts @@ -571,6 +571,120 @@ describe("column catalog response boundary", () => { expect(invalid.status).toBe("malformed"); }); + it("rejects revoked response arrays without throwing", () => { + const captured = provider(() => undefined); + for (const response of [ + (() => { + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + return { epoch, relations: revoked.proxy }; + })(), + (() => { + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + return { + epoch, + relations: [ + { + ...readyResponse().relations[0], + columns: revoked.proxy, + }, + readyResponse().relations[1], + ], + }; + })(), + ]) { + expect(() => + decodeSqlColumnCatalogBatchResponse( + captured, + request(), + response, + ) + ).not.toThrow(); + expect(decodeSqlColumnCatalogBatchResponse( + captured, + request(), + response, + )).toEqual({ + reason: "invalid-shape", + status: "malformed", + }); + } + }); + + it("rejects cross-variant relation fields", () => { + const captured = provider(() => undefined); + const invalid = [ + { + code: "unknown", + requestKey: "users", + retry: "never", + status: "loading", + }, + { + code: "unknown", + ...readyResponse().relations[0], + retry: "never", + }, + { + code: "unknown", + columns: [], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + retry: "never", + status: "failed", + }, + ]; + for (const relation of invalid) { + expect(decodeSqlColumnCatalogBatchResponse( + captured, + request(), + { + epoch, + relations: [relation, readyResponse().relations[1]], + }, + ).status).toBe("malformed"); + } + }); + + it("rejects NUL-delimited conflicting column identities", () => { + const result = decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + { + epoch, + relations: [ + { + columns: [ + { + columnEntityId: "same", + identifier: { quoted: false, value: "x\u0000y" }, + insertText: "z", + ordinal: 0, + }, + { + columnEntityId: "same", + identifier: { quoted: false, value: "x" }, + insertText: "y\u0000z", + ordinal: 0, + }, + ], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + readyResponse().relations[1], + ], + }, + ); + expect(result).toEqual({ + reason: "duplicate-column-entity-id", + status: "malformed", + }); + }); + it("rejects malformed relation and column state combinations", () => { const captured = provider(() => undefined); const request_ = request(); diff --git a/src/vnext/__tests__/column-completion.test.ts b/src/vnext/__tests__/column-completion.test.ts index b30e118..f86655c 100644 --- a/src/vnext/__tests__/column-completion.test.ts +++ b/src/vnext/__tests__/column-completion.test.ts @@ -3,6 +3,9 @@ import { composeSqlColumnCompletion, prepareSqlColumnCatalogRelations, } from "../column-completion.js"; +import { + MAX_COLUMN_BATCH_RELATIONS, +} from "../column-catalog-boundary.js"; import type { SqlColumnCatalogBatchOutcome, } from "../column-catalog-batch-coordinator.js"; @@ -91,6 +94,51 @@ function usable( } describe("column completion", () => { + it("bounds relation batches and reports the omitted bindings", () => { + const relations = Array.from( + { length: MAX_COLUMN_BATCH_RELATIONS + 1 }, + (_, index) => Object.freeze({ + alias: Object.freeze({ + quoted: false, + value: `t_${index}`, + }), + path: Object.freeze([ + Object.freeze({ + quoted: false, + value: `table_${index}`, + }), + ]), + range: Object.freeze({ from: index, to: index + 1 }), + }), + ); + const current = Object.freeze({ + ...site(), + relations: Object.freeze(relations), + }); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + + expect(prepared).toMatchObject({ + coverage: "partial", + references: { length: MAX_COLUMN_BATCH_RELATIONS }, + }); + expect(composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([]), + prepared, + providerId: "columns", + site: current, + })).toMatchObject({ + sources: [{ coverage: "partial", outcome: "ready" }], + value: { + isIncomplete: true, + issues: [{ reason: "query-binding-partial" }], + }, + }); + }); + it("batches only the relation selected by an alias qualifier", () => { const current = site([{ quoted: false, value: "u" }], "na"); const prepared = prepareSqlColumnCatalogRelations( diff --git a/src/vnext/__tests__/column-query-site.test.ts b/src/vnext/__tests__/column-query-site.test.ts index 0f075ef..1bed1a9 100644 --- a/src/vnext/__tests__/column-query-site.test.ts +++ b/src/vnext/__tests__/column-query-site.test.ts @@ -97,7 +97,7 @@ describe("recognizeSqlColumnQuerySite", () => { .toEqual(["u", "i", "o"]); }); - it("uses the innermost query block", () => { + it("includes correlated outer-query relations", () => { const result = ready( analyze( "SELECT * FROM outer_table o WHERE EXISTS (SELECT i.na| FROM inner_table i)", @@ -105,10 +105,61 @@ describe("recognizeSqlColumnQuerySite", () => { ); expect(result.qualifier[0]?.value).toBe("i"); - expect(result.relations).toHaveLength(1); - expect(result.relations[0]?.path.at(-1)?.value).toBe( - "inner_table", + expect(result.coverage).toBe("partial"); + expect(result.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(["inner_table", "outer_table"]); + }); + + it("does not correlate an ordinary derived table", () => { + const result = ready( + analyze( + "SELECT * FROM (SELECT i.na| FROM inner_table i) d JOIN outer_table o ON true", + ), ); + + expect(result.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(["inner_table"]); + }); + + it("limits correlation in a JOIN condition to prior relations", () => { + const result = ready( + analyze( + "SELECT * FROM users u JOIN orders o ON EXISTS (SELECT u.|) JOIN secrets s ON true", + ), + ); + + expect(result.relations.map((relation) => + relation.alias?.value + )).toEqual(["u", "o"]); + }); + + it("walks nested correlation without adopting a closed sibling", () => { + const nested = ready( + analyze( + "SELECT * FROM root_table r WHERE EXISTS (SELECT * FROM mid_table m WHERE EXISTS (SELECT r.|))", + ), + ); + expect(nested.relations.map((relation) => + relation.alias?.value + )).toEqual(["m", "r"]); + + const sibling = ready( + analyze( + "SELECT * FROM root_table r WHERE EXISTS (SELECT 1) AND EXISTS (SELECT r.|)", + ), + ); + expect(sibling.relations.map((relation) => + relation.alias?.value + )).toEqual(["r"]); + }); + + it("allows a parenthesized top-level query without an outer scope", () => { + const result = ready(analyze("(SELECT | FROM inner_table)")); + expect(result.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(["inner_table"]); }); it("keeps set-operation arms and JOIN visibility isolated", () => { @@ -236,6 +287,23 @@ describe("recognizeSqlColumnQuerySite", () => { issues: ["incomplete-relation"], relations: [], }); + expect(ready(analyze("SELECT | FROM users AS"))).toMatchObject({ + coverage: "partial", + issues: ["incomplete-relation"], + relations: [{ + alias: null, + path: [{ value: "users" }], + }], + }); + expect(ready(analyze("SELECT | FROM users AS WHERE true"))) + .toMatchObject({ + coverage: "partial", + issues: ["incomplete-relation"], + }); + expect(ready(analyze('SELECT | FROM users AS "u"')).relations[0]) + .toMatchObject({ + alias: { quoted: true, value: "u" }, + }); }); it("fails closed for malformed typed paths and line comments", () => { @@ -247,6 +315,22 @@ describe("recognizeSqlColumnQuerySite", () => { reason: "cursor-in-comment", status: "inactive", }); + expect(analyze("SELECT 1 -- comment|")).toMatchObject({ + reason: "cursor-in-comment", + status: "inactive", + }); + expect(analyze("SELECT 1 -- comment|\nFROM users")).toMatchObject({ + reason: "cursor-in-comment", + status: "inactive", + }); + expect( + analyze("SELECT 1 # comment|", { + dialect: BIGQUERY_SQL_RELATION_DIALECT, + }), + ).toMatchObject({ + reason: "cursor-in-comment", + status: "inactive", + }); }); it("bounds lexical work before relation analysis", () => { @@ -269,6 +353,11 @@ describe("recognizeSqlColumnQuerySite", () => { reason: "resource-limit", status: "unavailable", }); + expect(analyze(`SELECT (SELECT |) FROM base b${joins}`)) + .toMatchObject({ + reason: "resource-limit", + status: "unavailable", + }); }); it("rejects foreign contract values without inspecting them", () => { @@ -323,6 +412,30 @@ describe("recognizeSqlColumnQuerySite", () => { } }); + it("rejects a cursor outside its authenticated exact slot", () => { + const text = + "SELECT x FROM first_table WHERE ; SELECT y FROM second_table"; + const source = createIdentitySqlSource(text); + const index = buildSqlStatementIndex( + text, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const first = findSqlStatementSlot(index, 0, "right"); + const secondPosition = text.indexOf("y FROM"); + + expect( + recognizeSqlColumnQuerySite( + source, + first, + secondPosition, + POSTGRESQL_SQL_RELATION_DIALECT, + ), + ).toEqual({ + reason: "not-column-position", + status: "inactive", + }); + }); + it("preserves opaque statement failures", () => { const source = createIdentitySqlSource("DELIMITER $$"); const index = buildSqlStatementIndex( diff --git a/src/vnext/__tests__/hostile-data.test.ts b/src/vnext/__tests__/hostile-data.test.ts new file mode 100644 index 0000000..f5194d2 --- /dev/null +++ b/src/vnext/__tests__/hostile-data.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; +import { isDataArray } from "../types.js"; + +describe("hostile data arrays", () => { + it("recognizes arrays without invoking elements", () => { + expect(isDataArray([])).toBe(true); + expect(isDataArray({})).toBe(false); + }); + + it("contains revoked and hostile proxy traps", () => { + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + expect(isDataArray(revoked.proxy)).toBe(false); + }); +}); diff --git a/src/vnext/__tests__/local-relation-site.test.ts b/src/vnext/__tests__/local-relation-site.test.ts index 22b4425..1a1c56f 100644 --- a/src/vnext/__tests__/local-relation-site.test.ts +++ b/src/vnext/__tests__/local-relation-site.test.ts @@ -412,6 +412,34 @@ describe("local relation-site evidence", () => { ); }); + it("fails column analysis closed outside the prepared statement", () => { + const text = + "SELECT x FROM first_table WHERE ; SELECT y FROM second_table"; + const source = createIdentitySqlSource(text); + const index = buildSqlStatementIndex( + text, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const first = findSqlStatementSlot(index, 0, "right"); + const preparation = prepareSqlLocalRelationStatement( + source, + index, + first, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + if (preparation.status !== "ready") { + throw new Error("Expected exact first-statement preparation"); + } + + expect(analyzeSqlLocalColumnSite( + preparation.statement, + text.indexOf("y FROM"), + )).toEqual({ + reason: "not-column-position", + status: "inactive", + }); + }); + it("separates qualified sites from irrelevant CTE uncertainty", () => { const ready = expectReady( analyzeMarked( diff --git a/src/vnext/__tests__/namespace-catalog-boundary.test.ts b/src/vnext/__tests__/namespace-catalog-boundary.test.ts index 126695a..bf2cae6 100644 --- a/src/vnext/__tests__/namespace-catalog-boundary.test.ts +++ b/src/vnext/__tests__/namespace-catalog-boundary.test.ts @@ -119,6 +119,8 @@ describe("namespace catalog boundary", () => { const valid = request(); const sparse: unknown[] = []; sparse.length = 1; + const revoked = Proxy.revocable([], {}); + revoked.revoke(); const cases: unknown[] = [ null, { ...valid, extra: true }, @@ -133,8 +135,11 @@ describe("namespace catalog boundary", () => { { ...valid, limit: 1.5 }, { ...valid, prefix: { quoted: "no", value: "x" } }, { ...valid, prefix: { quoted: false, value: "x", extra: 1 } }, + { ...valid, qualifier: [{ quoted: false, value: "" }] }, + { ...valid, searchPaths: [[{ quoted: false, value: "" }]] }, { ...valid, qualifier: sparse }, { ...valid, searchPaths: sparse }, + { ...valid, qualifier: revoked.proxy }, { ...valid, qualifier: [{ quoted: false, value: "x".repeat(257) }] }, { ...valid, searchPaths: Array.from({ length: 33 }, () => []) }, ]; @@ -163,6 +168,11 @@ describe("namespace catalog boundary", () => { expect(createSqlNamespaceCatalogSearchRequest(value).status) .toBe("malformed"); } + expect(createSqlNamespaceCatalogSearchRequest({ + ...valid, + prefix: { quoted: false, value: "" }, + qualifier: [], + }).status).toBe("accepted"); }); it("decodes, deduplicates, orders, and freezes all container roles", () => { @@ -185,7 +195,9 @@ describe("namespace catalog boundary", () => { ); expect(decoded.status).toBe("accepted"); if (decoded.status !== "accepted" || - decoded.value.status !== "ready") return; + decoded.value.status !== "ready") { + throw new Error("Expected a ready namespace response"); + } expect(decoded.value.containers.map((value) => value.canonicalPath[0]?.role )).toEqual(["catalog", "dataset", "project", "schema"]); @@ -266,6 +278,74 @@ describe("namespace catalog boundary", () => { }); }); + it("orders raw identifier values by UTF-16 code units", () => { + const decoded = decodeSqlNamespaceCatalogSearchResponse( + captured(), + request(), + { + containers: [ + container("hash", "schema", "#"), + container("quote", "schema", "\""), + container("slash", "schema", "\\"), + container("control", "schema", "\u0000"), + ], + coverage: "complete", + epoch, + status: "ready", + }, + ); + expect(decoded).toMatchObject({ + status: "accepted", + value: { + containers: [ + { containerEntityId: "control" }, + { containerEntityId: "quote" }, + { containerEntityId: "hash" }, + { containerEntityId: "slash" }, + ], + }, + }); + }); + + it("compares canonical paths structurally when they contain NUL", () => { + const provider = captured(); + const base = { + containerEntityId: "same", + insertText: "value", + matchQuality: "exact", + }; + const decoded = decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + { + containers: [ + { + ...base, + canonicalPath: [{ + quoted: false, + role: "schema", + value: "a\u0000catalog:u:b", + }], + }, + { + ...base, + canonicalPath: [ + { quoted: false, role: "schema", value: "a" }, + { quoted: false, role: "catalog", value: "b" }, + ], + }, + ], + coverage: "complete", + epoch, + status: "ready", + }, + ); + expect(decoded).toEqual({ + reason: "duplicate-entity-id", + status: "malformed", + }); + }); + it("rejects malformed, conflicting, stale, and oversized responses", () => { const provider = captured(); const valid = { @@ -281,8 +361,15 @@ describe("namespace catalog boundary", () => { container("same", "schema", "other"), ], }; + const revoked = Proxy.revocable([], {}); + revoked.revoke(); const malformedContainers = [ { ...container("x", "schema", "x"), canonicalPath: [] }, + { ...container("x", "schema", "x"), canonicalPath: [{ + quoted: false, + role: "schema", + value: "", + }] }, { ...container("x", "schema", "x"), canonicalPath: [{ quoted: false, role: "table", @@ -304,6 +391,7 @@ describe("namespace catalog boundary", () => { ) }, conflicting, { ...valid, containers: [null] }, + { ...valid, containers: revoked.proxy }, { epoch, extra: true, status: "loading" }, { code: "bad", epoch, retry: "never", status: "failed" }, { code: "unknown", epoch, retry: "bad", status: "failed" }, diff --git a/src/vnext/__tests__/namespace-catalog-coordinator.bench.ts b/src/vnext/__tests__/namespace-catalog-coordinator.bench.ts index c42dbfd..47489ee 100644 --- a/src/vnext/__tests__/namespace-catalog-coordinator.bench.ts +++ b/src/vnext/__tests__/namespace-catalog-coordinator.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe } from "vitest"; +import { beforeAll, bench, describe } from "vitest"; import { createSqlNamespaceCatalogCoordinator, } from "../namespace-catalog-coordinator.js"; @@ -61,9 +61,11 @@ describe("namespace catalog coordinator", () => { const warm = owner(); const primed = warm.request(search).result; + beforeAll(async () => { + await primed; + }); bench("warm 100-container cache lookup", async () => { - await primed; await warm.request(search).result; }); }); diff --git a/src/vnext/__tests__/namespace-catalog-coordinator.test.ts b/src/vnext/__tests__/namespace-catalog-coordinator.test.ts index b603fee..3760d80 100644 --- a/src/vnext/__tests__/namespace-catalog-coordinator.test.ts +++ b/src/vnext/__tests__/namespace-catalog-coordinator.test.ts @@ -177,6 +177,83 @@ describe("namespace catalog coordinator", () => { await Promise.resolve(); }); + it("preserves the newest request across reentrant abort handlers", async () => { + const signals: AbortSignal[] = []; + let owner: ReturnType["owner"] | null = null; + const reentrant: { + ticket: + | ReturnType["owner"]["request"]> + | null; + } = { ticket: null }; + const configured = setup((_request, signal) => { + signals.push(signal); + if (signals.length === 1) { + signal.addEventListener("abort", () => { + reentrant.ticket = owner?.request(input("reentrant")) ?? null; + }, { once: true }); + } + return new Promise(() => undefined); + }); + owner = configured.owner; + const first = owner.request(input("first")); + const interrupted = owner.request(input("interrupted")); + + await expect(first.result).resolves.toEqual({ + status: "superseded", + }); + await expect(interrupted.result).resolves.toEqual({ + status: "superseded", + }); + expect(signals.map((signal) => signal.aborted)).toEqual([ + true, + false, + ]); + + const newest = owner.request(input("newest")); + expect(signals.map((signal) => signal.aborted)).toEqual([ + true, + true, + false, + ]); + await expect(reentrant.ticket?.result).resolves.toEqual({ + status: "superseded", + }); + newest.cancel(); + }); + + it.each(["owner", "coordinator"] as const)( + "does not resurrect work after reentrant %s disposal", + async (target) => { + const signals: AbortSignal[] = []; + let dispose = (): void => {}; + const configured = setup((_request, signal) => { + signals.push(signal); + if (signals.length === 1) { + signal.addEventListener("abort", () => dispose(), { + once: true, + }); + } + return new Promise(() => undefined); + }); + dispose = target === "owner" + ? configured.owner.dispose + : configured.coordinator.dispose; + const first = configured.owner.request(input("first")); + const interrupted = configured.owner.request(input("interrupted")); + + await expect(first.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + await expect(interrupted.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + expect(signals).toHaveLength(1); + expect(signals[0]?.aborted).toBe(true); + }, + ); + it("isolates owners and settles disposal", async () => { const pending: Array>> = []; const { coordinator, owner } = setup(() => { diff --git a/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts b/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts index 89f4460..672f973 100644 --- a/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts +++ b/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts @@ -184,6 +184,130 @@ describe("node-sql-parser query binding normalization", () => { } }); + it("preserves quoted CTE identity and case semantics", () => { + const cteStatement = { + from: [relation("events")], + type: "select", + }; + const quoted = normalize( + { + from: [relation("Recent")], + type: "select", + with: [ + { + name: { type: "default", value: "Recent" }, + stmt: cteStatement, + }, + ], + }, + 'WITH "Recent" AS (SELECT * FROM events) SELECT * FROM "Recent"', + ); + expect(quoted).toMatchObject({ + model: { + bindings: [ + { source: { kind: "cte", name: { quoted: true, value: "Recent" } } }, + { source: { kind: "named" } }, + ], + }, + status: "ready", + }); + + const differentlyQuoted = normalize( + { + from: [relation("recent")], + type: "select", + with: [ + { + name: { type: "default", value: "Recent" }, + stmt: cteStatement, + }, + ], + }, + 'WITH "Recent" AS (SELECT * FROM events) SELECT * FROM recent', + ); + expect(differentlyQuoted).toMatchObject({ + model: { + bindings: [ + { source: { kind: "named", path: [{ value: "recent" }] } }, + { source: { kind: "named" } }, + ], + }, + status: "ready", + }); + }); + + it("applies parent and preceding-sibling CTE visibility per block", () => { + const first = { + from: [relation("later")], + type: "select", + }; + const later = { + from: [relation("events")], + type: "select", + }; + const result = normalize( + { + from: [relation("first")], + type: "select", + with: [ + { name: { value: "first" }, stmt: first }, + { name: { value: "later" }, stmt: later }, + ], + }, + "WITH first AS (SELECT * FROM later), " + + "later AS (SELECT * FROM events) SELECT * FROM first", + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte", name: { value: "first" } } }, + { + owner: 1, + source: { kind: "named", path: [{ value: "later" }] }, + }, + { + owner: 2, + source: { kind: "named", path: [{ value: "events" }] }, + }, + ], + }, + status: "ready", + }); + }); + + it("resolves a nested WITH inside its derived query block", () => { + const nestedCte = { + from: [relation("events")], + type: "select", + }; + const nested = { + from: [relation("local")], + type: "select", + with: [{ name: { value: "local" }, stmt: nestedCte }], + }; + const result = normalize( + { + from: [{ as: "d", expr: { ast: nested, parentheses: true } }], + type: "select", + }, + "SELECT * FROM (WITH local AS (SELECT * FROM events) " + + "SELECT * FROM local) d", + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { block: 1, kind: "derived" } }, + { owner: 1, source: { kind: "cte", name: { value: "local" } } }, + { + owner: 2, + source: { kind: "named", path: [{ value: "events" }] }, + }, + ], + }, + status: "ready", + }); + }); + it("normalizes BigQuery multipart paths and QUALIFY", () => { const text = "SELECT a.id FROM `project.dataset.users` a " + @@ -305,9 +429,23 @@ describe("node-sql-parser query binding normalization", () => { { from: [relation("users", "u")], type: "select" }, "SELECT * FROM users", ); - for (const result of [lexicalOnly, astOnly]) { + const differentValues = normalize( + { from: [relation("users", "x")], type: "select" }, + "SELECT * FROM users u", + ); + const differentQuotes = normalize( + { from: [relation("users", "U")], type: "select" }, + 'SELECT * FROM users AS "u"', + ); + for (const result of [ + lexicalOnly, + astOnly, + differentValues, + differentQuotes, + ]) { expect(result).toMatchObject({ model: { + bindings: [{ alias: null }], coverage: { relationBindings: "partial" }, issues: [{ code: "unsupported-relation-source" }], }, @@ -316,6 +454,189 @@ describe("node-sql-parser query binding normalization", () => { } }); + it("never publishes AST relation names absent from the SQL text", () => { + const table = normalize( + { from: [relation("events")], type: "select" }, + "SELECT * FROM users", + ); + const schema = normalize( + { + from: [{ as: null, db: "private", table: "users" }], + type: "select", + }, + "SELECT * FROM public.users", + ); + for (const result of [table, schema]) { + expect(result).toMatchObject({ + model: { + bindings: [ + { + source: { + kind: "unknown", + reason: "unsupported-relation-source", + }, + }, + ], + coverage: { relationBindings: "partial" }, + issues: [{ code: "unsupported-relation-source" }], + }, + status: "ready", + }); + } + expect( + normalize( + { from: [relation("private.dataset.users")], type: "select" }, + "SELECT * FROM `public.dataset.users`", + BQ, + ), + ).toMatchObject({ + model: { + bindings: [{ source: { kind: "unknown" } }], + coverage: { relationBindings: "partial" }, + }, + status: "ready", + }); + }); + + it("keeps ON and USING join visibility aligned with relation sites", () => { + const onText = "SELECT a.id, b.id FROM a JOIN b ON a.id=b.id"; + const onResult = normalize( + { + from: [ + relation("a"), + { ...relation("b"), join: "JOIN", on: {} }, + ], + type: "select", + }, + onText, + ); + const usingText = "SELECT * FROM a JOIN b USING (id)"; + const usingResult = normalize( + { + from: [ + relation("a"), + { ...relation("b"), join: "JOIN" }, + ], + type: "select", + }, + usingText, + ); + for (const [result, position] of [ + [onResult, onText.indexOf("a.id", onText.indexOf("ON"))], + [usingResult, usingText.indexOf("id")], + ] as const) { + expect(result.status).toBe("ready"); + if (result.status === "ready") { + expect(result.model.coverage.visibility).toBe("complete"); + expect(visibleSqlRelationBindingsAt(result.model, position)).toMatchObject({ + bindings: [{}, {}], + region: { kind: "join-condition" }, + status: "ready", + }); + } + } + }); + + it("does not assign a later join condition to an earlier relation", () => { + const text = "SELECT * FROM a JOIN b JOIN c ON c.id=b.id"; + const result = normalize( + { + from: [ + relation("a"), + { ...relation("b"), join: "JOIN" }, + { ...relation("c"), join: "JOIN", on: {} }, + ], + type: "select", + }, + text, + ); + expect(result.status).toBe("ready"); + if (result.status === "ready") { + const regions = result.model.regions.filter((region) => + region.kind === "join-condition" + ); + expect(regions).toHaveLength(1); + expect(visibleSqlRelationBindingsAt( + result.model, + text.indexOf("c.id"), + )).toMatchObject({ + bindings: [{}, {}, {}], + status: "ready", + }); + } + }); + + it.each([ + ["WHERE true", { where: {} }], + ["GROUP BY a.id", { groupby: [{ expr: {} }] }], + ["HAVING true", { having: {} }], + ["QUALIFY true", { qualify: {} }], + ["ORDER BY a.id", { orderby: [{ expr: {} }] }], + ["LIMIT 1", { limit: {} }], + ])( + "stops a conditionless join before %s", + (suffix, clause) => { + const result = normalize( + { + ...clause, + from: [ + relation("a"), + { ...relation("b"), join: "CROSS JOIN" }, + ], + type: "select", + }, + `SELECT * FROM a CROSS JOIN b ${suffix}`, + ); + expect(result.status).toBe("ready"); + if (result.status === "ready") { + expect( + result.model.regions.some((region) => + region.kind === "join-condition" + ), + ).toBe(false); + } + }, + ); + + it.each([ + "WHERE true", + "GROUP BY a.id", + "HAVING true", + "QUALIFY true", + "ORDER BY a.id", + "LIMIT 1", + ])("ends an ON region before %s", (suffix) => { + const text = `SELECT * FROM a JOIN b ON true ${suffix}`; + const result = normalize( + { + from: [ + relation("a"), + { ...relation("b"), join: "JOIN", on: {} }, + ], + type: "select", + }, + text, + ); + expect(result.status).toBe("ready"); + if (result.status === "ready") { + const join = result.model.regions.find((region) => + region.kind === "join-condition" + ); + expect(join?.range.to).toBe( + text.indexOf(suffix.split(" ")[0] ?? suffix), + ); + } + }); + + it("emits compound query block kinds", () => { + for (const type of ["union", "intersect", "except"]) { + expect(normalize({ type }, "SELECT 1")).toMatchObject({ + model: { blocks: [{ kind: "compound" }] }, + status: "ready", + }); + } + }); + it("preserves PostgreSQL quoted path components", () => { const result = normalize( { @@ -409,6 +730,15 @@ describe("node-sql-parser query binding hostile boundaries", () => { reason: "malformed-ast", status: "unavailable", }); + + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + expect( + normalize({ from: revoked.proxy, type: "select" }, "SELECT 1"), + ).toEqual({ + reason: "malformed-ast", + status: "unavailable", + }); }); it("rejects malformed known AST properties and CTE entries", () => { @@ -444,6 +774,85 @@ describe("node-sql-parser query binding hostile boundaries", () => { } }); + it("rejects CTE structures that change across safe inspections", () => { + const child = { type: "select" }; + const item = { name: "x", stmt: child }; + function changingRoot(secondWith: unknown) { + let reads = 0; + return new Proxy( + { type: "select", with: [item] }, + { + getOwnPropertyDescriptor(target, key) { + const descriptor = Reflect.getOwnPropertyDescriptor(target, key); + if (key !== "with" || descriptor === undefined) { + return descriptor; + } + reads += 1; + return { ...descriptor, value: reads === 1 ? [item] : secondWith }; + }, + }, + ); + } + for (const root of [changingRoot({}), changingRoot([])]) { + expect( + normalize(root, "WITH x AS (SELECT 1) SELECT 1"), + ).toEqual({ reason: "malformed-ast", status: "unavailable" }); + } + + let itemReads = 0; + const changingItems = new Proxy([item], { + getOwnPropertyDescriptor(target, key) { + const descriptor = Reflect.getOwnPropertyDescriptor(target, key); + if (key !== "0" || descriptor === undefined) { + return descriptor; + } + itemReads += 1; + return { ...descriptor, value: itemReads === 1 ? item : null }; + }, + }); + expect( + normalize( + { type: "select", with: changingItems }, + "WITH x AS (SELECT 1) SELECT 1", + ), + ).toEqual({ reason: "malformed-ast", status: "unavailable" }); + + function changingChild() { + let reads = 0; + return new Proxy( + { type: "select", with: [] }, + { + getOwnPropertyDescriptor(target, key) { + const descriptor = Reflect.getOwnPropertyDescriptor(target, key); + if (key !== "with" || descriptor === undefined) { + return descriptor; + } + reads += 1; + return { ...descriptor, value: reads === 1 ? [] : {} }; + }, + }, + ); + } + expect( + normalize( + { + type: "select", + with: [{ name: "x", stmt: changingChild() }], + }, + "WITH x AS (SELECT 1) SELECT 1", + ), + ).toEqual({ reason: "malformed-ast", status: "unavailable" }); + expect( + normalize( + { + from: [{ as: "d", expr: { ast: changingChild() } }], + type: "select", + }, + "SELECT * FROM (SELECT 1) d", + ), + ).toEqual({ reason: "malformed-ast", status: "unavailable" }); + }); + it("handles comments, incomplete relation sites, and nesting limits", () => { expect( normalize( diff --git a/src/vnext/__tests__/query-binding-model.bench.ts b/src/vnext/__tests__/query-binding-model.bench.ts index 0231e49..ae742fa 100644 --- a/src/vnext/__tests__/query-binding-model.bench.ts +++ b/src/vnext/__tests__/query-binding-model.bench.ts @@ -1,6 +1,10 @@ import { bench, describe } from "vitest"; import { createSqlQueryBindingModel, + MAX_QUERY_BINDING_BLOCKS, + MAX_QUERY_BINDINGS, + MAX_QUERY_BINDING_SCOPES, + MAX_QUERY_VISIBILITY_REGIONS, resolveSqlRelationQualifier, visibleSqlRelationBindingsAt, } from "../query-binding-model.js"; @@ -62,6 +66,66 @@ const input = { const model = createSqlQueryBindingModel(TEXT, AUTHORITY, input); +const MAXIMUM_TEXT = "x".repeat(MAX_QUERY_VISIBILITY_REGIONS * 2); +const maximumBlocks = Array.from( + { length: MAX_QUERY_BINDING_BLOCKS }, + (_, index) => ({ + baseScope: 0, + kind: "select", + parentBlock: index === 0 ? null : index - 1, + range: { from: 0, to: MAXIMUM_TEXT.length }, + }), +); +const maximumBindings = Array.from( + { length: MAX_QUERY_BINDINGS }, + (_, index) => ({ + alias: null, + owner: index % MAX_QUERY_BINDING_BLOCKS, + range: { from: 0, to: 1 }, + source: { + kind: "named", + path: [component(`relation_${index}`)], + }, + }), +); +const maximumScopes: { + addedBinding: number | null; + parentScope: number | null; +}[] = [ + { addedBinding: null, parentScope: null }, + ...maximumBindings.map((_binding, index) => ({ + addedBinding: index, + parentScope: index, + })), +]; +while (maximumScopes.length < MAX_QUERY_BINDING_SCOPES) { + maximumScopes.push({ + addedBinding: null, + parentScope: maximumScopes.length - 1, + }); +} +const maximumInput = { + bindings: maximumBindings, + blocks: maximumBlocks, + coverage: { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }, + issues: [], + regions: Array.from( + { length: MAX_QUERY_VISIBILITY_REGIONS }, + (_, index) => ({ + block: MAX_QUERY_BINDING_BLOCKS - 1, + kind: "where", + range: { from: index * 2, to: index * 2 + 1 }, + scope: MAX_QUERY_BINDING_SCOPES - 1, + }), + ), + scopes: maximumScopes, + statementRange: { from: 0, to: MAXIMUM_TEXT.length }, +}; + function asciiEqual( left: SqlIdentifierComponent, right: SqlIdentifierComponent, @@ -74,6 +138,10 @@ describe("query binding model", () => { createSqlQueryBindingModel(TEXT, AUTHORITY, input); }); + bench("validate maximum relationship dimensions", () => { + createSqlQueryBindingModel(MAXIMUM_TEXT, AUTHORITY, maximumInput); + }); + bench("walk 1,000 visible bindings", () => { visibleSqlRelationBindingsAt(model, 5_000); }); diff --git a/src/vnext/__tests__/query-binding-model.test.ts b/src/vnext/__tests__/query-binding-model.test.ts index dee04a1..7126094 100644 --- a/src/vnext/__tests__/query-binding-model.test.ts +++ b/src/vnext/__tests__/query-binding-model.test.ts @@ -4,6 +4,9 @@ import { isSqlQueryBindingModel, isSqlQueryBindingModelError, MAX_QUERY_BINDING_BLOCKS, + MAX_QUERY_BINDINGS, + MAX_QUERY_BINDING_SCOPES, + MAX_QUERY_VISIBILITY_REGIONS, resolveSqlRelationQualifier, sqlQueryBindingModelMatches, visibleSqlRelationBindingsAt, @@ -717,6 +720,117 @@ describe("query binding model validation", () => { expectError(() => model({ bindings, blocks }), "invalid-model"); }); + it("rejects a derived binding that points at a sibling outside its range", () => { + const text = "x".repeat(20); + expectError( + () => + createSqlQueryBindingModel(text, {}, { + bindings: [ + { + alias: null, + owner: 0, + range: range(1, 5), + source: { block: 2, kind: "derived" }, + }, + ], + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(0, 20), + }, + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(1, 5), + }, + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(10, 15), + }, + ], + coverage: completeCoverage(), + issues: [], + regions: [], + scopes: [ + { addedBinding: null, parentScope: null }, + { addedBinding: 0, parentScope: 0 }, + ], + statementRange: range(0, text.length), + }), + "invalid-model", + ); + }); + + it("bounds maximum relationship validation work", () => { + const text = "x".repeat(MAX_QUERY_VISIBILITY_REGIONS * 2); + const blocks = Array.from( + { length: MAX_QUERY_BINDING_BLOCKS }, + (_, index) => ({ + baseScope: 0, + kind: "select", + parentBlock: index === 0 ? null : index - 1, + range: range(0, text.length), + }), + ); + const bindings = Array.from( + { length: MAX_QUERY_BINDINGS }, + (_, index) => ({ + alias: null, + owner: index % MAX_QUERY_BINDING_BLOCKS, + range: range(0, 1), + source: { + kind: "named", + path: [component(`r${index}`)], + }, + }), + ); + const scopes: { + addedBinding: number | null; + parentScope: number | null; + }[] = [ + { addedBinding: null, parentScope: null }, + ...bindings.map((_binding, index) => ({ + addedBinding: index, + parentScope: index, + })), + ]; + while (scopes.length < MAX_QUERY_BINDING_SCOPES) { + scopes.push({ + addedBinding: null, + parentScope: scopes.length - 1, + }); + } + const regions = Array.from( + { length: MAX_QUERY_VISIBILITY_REGIONS }, + (_, index) => ({ + block: MAX_QUERY_BINDING_BLOCKS - 1, + kind: "where", + range: range(index * 2, index * 2 + 1), + scope: MAX_QUERY_BINDING_SCOPES - 1, + }), + ); + const start = performance.now(); + const result = createSqlQueryBindingModel(text, {}, { + bindings, + blocks, + coverage: completeCoverage(), + issues: [], + regions, + scopes, + statementRange: range(0, text.length), + }); + const duration = performance.now() - start; + + expect(result.bindings).toHaveLength(MAX_QUERY_BINDINGS); + expect(result.regions).toHaveLength(MAX_QUERY_VISIBILITY_REGIONS); + expect(duration).toBeLessThan(500); + }); + it("requires partial relation coverage for unknown sources", () => { const bindings = [ { diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index fd19472..145d2e8 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -1394,6 +1394,59 @@ describe("column completion session integration", () => { service.dispose(); }); + it("completes a qualified correlated outer relation", async () => { + const requests: Parameters< + SqlColumnCatalogProvider["loadColumns"] + >[0][] = []; + const service = serviceWithColumns(async (request) => { + requests.push(request); + const relation = request.relations[0]; + if (!relation) throw new Error("Expected the correlated relation"); + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: [{ + columns: [{ + columnEntityId: "users:id", + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: "users", + requestKey: relation.requestKey, + status: "ready", + }], + }; + }); + const text = + "SELECT * FROM users u WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.)"; + const position = text.indexOf("u.)") + 2; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:correlated" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + + await expect(session.complete({ + position, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + status: "ready", + value: { + items: [{ label: "id" }], + }, + }); + expect(requests).toHaveLength(1); + expect(requests[0]?.relations).toEqual([{ + path: [{ quoted: false, value: "users" }], + requestKey: "binding:1", + }]); + service.dispose(); + }); + it("never resolves a visible CTE as a physical relation", async () => { let calls = 0; const service = serviceWithColumns(async () => { @@ -1463,6 +1516,47 @@ describe("column completion session integration", () => { } }); + it("charges column analysis against the total response budget", async () => { + vi.useFakeTimers(); + const performanceNow = vi.spyOn(performance, "now") + .mockReturnValueOnce(0) + .mockReturnValue(30); + try { + const service = serviceWithColumns(() => new Promise(() => {})); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:column-total-budget" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + let settled = false; + const completion = session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + void completion.then(() => { + settled = true; + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(9); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + await expect(completion).resolves.toMatchObject({ + status: "ready", + value: { + issues: [{ reason: "column-catalog-loading" }], + }, + }); + service.dispose(); + } finally { + performanceNow.mockRestore(); + vi.useRealTimers(); + } + }); + it("bounds provider-declared column loading to one automatic retry", async () => { vi.useFakeTimers(); try { diff --git a/src/vnext/codemirror/__tests__/sql-editor.test.ts b/src/vnext/codemirror/__tests__/sql-editor.test.ts index 4b32cdd..063c6aa 100644 --- a/src/vnext/codemirror/__tests__/sql-editor.test.ts +++ b/src/vnext/codemirror/__tests__/sql-editor.test.ts @@ -1484,9 +1484,106 @@ describe("sqlEditor", () => { view.dispatch({}); await vi.waitFor(() => { expect(harness.completeSignals[0]?.aborted).toBe(true); + expect(currentCompletions(view.state)).toEqual([]); }); }); + it("restarts external sources after the SQL gate closes stale options", async () => { + let allowed = true; + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const support = sqlEditor({ + autocomplete: { + externalSources: [() => ({ + from: 16, + options: [{ label: "external_variable" }], + })], + isCompletionPositionAllowed: () => allowed, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + expect(currentCompletions(view.state).map((item) => item.label).sort()) + .toEqual(["external_variable", "users"]); + + allowed = false; + view.dispatch({}); + await vi.waitFor(() => { + expect(currentCompletions(view.state).map((item) => item.label)) + .toEqual(["external_variable"]); + }); + }); + + it("closes SQL options denied while the provider is resolving", async () => { + const runtime = controlledRuntime(); + let allowed = true; + let resolveResult: + | ((result: SqlCompletionResult) => void) + | undefined; + let revision: SqlRevision | null = null; + const harness = fakeService((currentRevision) => { + revision = currentRevision; + return new Promise((resolve) => { + resolveResult = resolve; + }); + }); + const support = createSqlEditorInternal({ + autocomplete: { + closeOnBlur: false, + isCompletionPositionAllowed: () => allowed, + }, + initialContext: context(), + service: harness.service, + }, runtime.runtime); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await vi.waitFor(() => + expect(harness.completeSignals).toHaveLength(1) + ); + if (revision === null) { + throw new Error("Expected deferred completion revision"); + } + allowed = false; + resolveResult?.(readyResult(revision, [completionItem()])); + await vi.waitFor(() => expect(runtime.queued).toHaveLength(1)); + + runtime.queued[0]?.(); + expect(runtime.closes).toEqual([view]); + }); + + it("does not let a stale gate-close task affect a newer editor state", async () => { + const runtime = controlledRuntime(); + let allowed = true; + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const support = createSqlEditorInternal({ + autocomplete: { + closeOnBlur: false, + isCompletionPositionAllowed: () => allowed, + }, + initialContext: context(), + service: harness.service, + }, runtime.runtime); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + allowed = false; + view.dispatch({}); + expect(runtime.queued).toHaveLength(1); + + view.dispatch({ selection: { anchor: 15 } }); + runtime.queued[0]?.(); + expect(runtime.closes).toEqual([]); + }); + it("disposes rich info when the position gate flips false", async () => { let allowed = true; const destroys: Array> = []; diff --git a/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts index 8fd02df..3c9fd35 100644 --- a/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts +++ b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts @@ -3,7 +3,7 @@ import { startCompletion, } from "@codemirror/autocomplete"; import { EditorView } from "@codemirror/view"; -import { expect, test } from "vitest"; +import { expect, onTestFinished, test } from "vitest"; import { createSqlLanguageService, duckdbDialect, @@ -13,6 +13,7 @@ import { sqlEditor } from "../index.js"; test("vNext completion gate routes unmatched template EOF to external sources", async () => { const parent = document.createElement("div"); document.body.append(parent); + onTestFinished(() => parent.remove()); let catalogSearches = 0; const service = createSqlLanguageService({ catalog: { @@ -46,6 +47,7 @@ test("vNext completion gate routes unmatched template EOF to external sources", }, dialects: [duckdbDialect()], }); + onTestFinished(() => service.dispose()); const support = sqlEditor({ autocomplete: { externalSources: [(context) => ({ @@ -77,21 +79,19 @@ test("vNext completion gate routes unmatched template EOF to external sources", parent, selection: { anchor: 15 }, }); + onTestFinished(() => view.destroy()); expect(startCompletion(view)).toBe(true); await expect.poll(() => currentCompletions(view.state).map((item) => item.label) ).toEqual(["python_variable"]); expect(catalogSearches).toBe(0); - - view.destroy(); - service.dispose(); - parent.remove(); }); test("vNext completion gate permits normal SQL in a browser editor", async () => { const parent = document.createElement("div"); document.body.append(parent); + onTestFinished(() => parent.remove()); const service = createSqlLanguageService({ catalog: { id: "browser-catalog", @@ -121,6 +121,7 @@ test("vNext completion gate permits normal SQL in a browser editor", async () => }, dialects: [duckdbDialect()], }); + onTestFinished(() => service.dispose()); const support = sqlEditor({ autocomplete: { isCompletionPositionAllowed: () => true, @@ -141,20 +142,88 @@ test("vNext completion gate permits normal SQL in a browser editor", async () => parent, selection: { anchor: documentText.length }, }); + onTestFinished(() => view.destroy()); expect(startCompletion(view)).toBe(true); await expect.poll(() => currentCompletions(view.state).map((item) => item.label) ).toContain("users"); +}); + +test("vNext completion gate preserves external sources when it closes SQL options", async () => { + const parent = document.createElement("div"); + document.body.append(parent); + onTestFinished(() => parent.remove()); + let allowed = true; + const service = createSqlLanguageService({ + catalog: { + id: "browser-gate-transition", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "ready" }, + relations: [{ + canonicalPath: [ + { + quoted: false, + role: "schema", + value: "main", + }, + { + quoted: false, + role: "relation", + value: "users", + }, + ], + completionPathStart: 1, + entityId: "main.users", + matchQuality: "exact", + relationKind: "table", + }], + status: "ready" as const, + }), + }, + dialects: [duckdbDialect()], + }); + onTestFinished(() => service.dispose()); + const support = sqlEditor({ + autocomplete: { + externalSources: [(context) => ({ + from: context.pos, + options: [{ label: "python_variable" }], + })], + isCompletionPositionAllowed: () => allowed, + }, + initialContext: { + catalog: { scope: "browser-gate-transition" }, + dialect: "duckdb", + }, + service, + }); + const documentText = "SELECT * FROM us"; + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + selection: { anchor: documentText.length }, + }); + onTestFinished(() => view.destroy()); - view.destroy(); - service.dispose(); - parent.remove(); + expect(startCompletion(view)).toBe(true); + await expect.poll(() => + currentCompletions(view.state).map((item) => item.label).sort() + ).toEqual(["python_variable", "users"]); + + allowed = false; + view.dispatch({}); + await expect.poll(() => + currentCompletions(view.state).map((item) => item.label) + ).toEqual(["python_variable"]); }); test("vNext editor applies a batched column completion", async () => { const parent = document.createElement("div"); document.body.append(parent); + onTestFinished(() => parent.remove()); let columnCalls = 0; const service = createSqlLanguageService({ catalog: { @@ -192,6 +261,7 @@ test("vNext editor applies a batched column completion", async () => { }, dialects: [duckdbDialect()], }); + onTestFinished(() => service.dispose()); const support = sqlEditor({ initialContext: { catalog: { scope: "browser-columns" }, @@ -206,6 +276,7 @@ test("vNext editor applies a batched column completion", async () => { parent, selection: { anchor: "SELECT u.na".length }, }); + onTestFinished(() => view.destroy()); expect(startCompletion(view)).toBe(true); await expect.poll(() => @@ -215,15 +286,12 @@ test("vNext editor applies a batched column completion", async () => { })) ).toEqual([{ label: "name", type: "property" }]); expect(columnCalls).toBe(1); - - view.destroy(); - service.dispose(); - parent.remove(); }); test("vNext editor exposes namespace containers at relation sites", async () => { const parent = document.createElement("div"); document.body.append(parent); + onTestFinished(() => parent.remove()); let namespaceCalls = 0; const service = createSqlLanguageService({ dialects: [duckdbDialect()], @@ -249,6 +317,7 @@ test("vNext editor exposes namespace containers at relation sites", async () => }, }, }); + onTestFinished(() => service.dispose()); const support = sqlEditor({ initialContext: { catalog: { scope: "browser-namespaces" }, @@ -263,6 +332,7 @@ test("vNext editor exposes namespace containers at relation sites", async () => parent, selection: { anchor: documentText.length }, }); + onTestFinished(() => view.destroy()); expect(startCompletion(view)).toBe(true); await expect.poll(() => @@ -272,8 +342,4 @@ test("vNext editor exposes namespace containers at relation sites", async () => })) ).toEqual([{ label: "main", type: "namespace" }]); expect(namespaceCalls).toBe(1); - - view.destroy(); - service.dispose(); - parent.remove(); }); diff --git a/src/vnext/codemirror/browser_tests/statement-gutter.test.ts b/src/vnext/codemirror/browser_tests/statement-gutter.test.ts index 147d763..2ba440b 100644 --- a/src/vnext/codemirror/browser_tests/statement-gutter.test.ts +++ b/src/vnext/codemirror/browser_tests/statement-gutter.test.ts @@ -1,5 +1,5 @@ import { EditorView } from "@codemirror/view"; -import { expect, test } from "vitest"; +import { expect, onTestFinished, test } from "vitest"; import { createSqlLanguageService, duckdbDialect, @@ -10,9 +10,11 @@ test("vNext statement gutter follows the current statement", async () => { const parent = document.createElement("div"); parent.style.height = "240px"; document.body.append(parent); + onTestFinished(() => parent.remove()); const service = createSqlLanguageService({ dialects: [duckdbDialect()], }); + onTestFinished(() => service.dispose()); const support = sqlEditor({ initialContext: { dialect: "duckdb" }, service, @@ -25,6 +27,7 @@ test("vNext statement gutter follows the current statement", async () => { parent, selection: { anchor: documentText.length }, }); + onTestFinished(() => view.destroy()); view.focus(); await expect.poll(() => @@ -48,10 +51,6 @@ test("vNext statement gutter follows the current statement", async () => { marker.classList.contains("cm-sql-statement-marker-active") ); }).toBe(0); - - view.destroy(); - service.dispose(); - parent.remove(); }); test("vNext statement gutter virtualizes a tall focused editor", async () => { @@ -62,9 +61,11 @@ test("vNext statement gutter virtualizes a tall focused editor", async () => { "rgb(1, 2, 3)", ); document.body.append(parent); + onTestFinished(() => parent.remove()); const service = createSqlLanguageService({ dialects: [duckdbDialect()], }); + onTestFinished(() => service.dispose()); const support = sqlEditor({ initialContext: { dialect: "duckdb" }, service, @@ -79,6 +80,7 @@ test("vNext statement gutter virtualizes a tall focused editor", async () => { extensions: support.extension, parent, }); + onTestFinished(() => view.destroy()); expect( view.dom.querySelectorAll(".cm-sql-statement-marker"), @@ -115,8 +117,4 @@ test("vNext statement gutter virtualizes a tall focused editor", async () => { expect( view.dom.querySelectorAll(".cm-sql-statement-marker").length, ).toBeLessThan(200); - - view.destroy(); - service.dispose(); - parent.remove(); }); diff --git a/src/vnext/codemirror/sql-editor.ts b/src/vnext/codemirror/sql-editor.ts index efd8366..9171149 100644 --- a/src/vnext/codemirror/sql-editor.ts +++ b/src/vnext/codemirror/sql-editor.ts @@ -130,6 +130,7 @@ export interface SqlEditorRuntime { interface CompletionCapture { readonly contextGeneration: number; readonly document: Text; + readonly embeddedRegions: readonly SqlEmbeddedRegion[]; readonly selection: EditorSelection; } @@ -287,6 +288,7 @@ export function createSqlEditorInternal< #contextGeneration = 0; #destroyed = false; #disposingInfo = false; + #completionPositionAllowed: boolean; #hasEmbeddedRegions: boolean; #info: ActiveCompletionInfo | null = null; #intent: CompletionIntent | null = null; @@ -308,6 +310,11 @@ export function createSqlEditorInternal< text: view.state.doc.toString(), }); this.#hasEmbeddedRegions = initialRegions.length > 0; + this.#completionPositionAllowed = + this.#completionPositionIsAllowed( + view.state, + view.state.selection.main.head, + ); this.#lastCompletionStatus = completionStatus(view.state); this.#lastSelectedCompletion = selectedCompletion(view.state); this.#lastSelectedCompletionIndex = selectedCompletionIndex( @@ -389,10 +396,21 @@ export function createSqlEditorInternal< !this.#destroyed && capture.contextGeneration === this.#contextGeneration && capture.document === state.doc && + capture.embeddedRegions === state.field(embeddedRegionsField) && capture.selection.eq(state.selection) ); } + #capture(): CompletionCapture { + const state = this.#view.state; + return { + contextGeneration: this.#contextGeneration, + document: state.doc, + embeddedRegions: state.field(embeddedRegionsField), + selection: state.selection, + }; + } + #completionPositionIsAllowed( state: EditorState, position: number, @@ -417,6 +435,31 @@ export function createSqlEditorInternal< }); } + #scheduleCompletionGateClose(capture: CompletionCapture): void { + runtime.queueMicrotask(() => { + if ( + !this.#captureIsCurrent(capture) || + this.#completionPositionIsAllowed( + this.#view.state, + this.#view.state.selection.main.head, + ) + ) { + return; + } + runtime.closeCompletion(this.#view); + if ( + externalSources.length > 0 && + this.#captureIsCurrent(capture) && + !this.#completionPositionIsAllowed( + this.#view.state, + this.#view.state.selection.main.head, + ) + ) { + runtime.startCompletion(this.#view); + } + }); + } + #scheduleRefresh(capture: CompletionCapture): void { if ( this.#refreshScheduled || @@ -572,11 +615,7 @@ export function createSqlEditorInternal< ) { return null; } - const capture: CompletionCapture = { - contextGeneration: this.#contextGeneration, - document: this.#view.state.doc, - selection: this.#view.state.selection, - }; + const capture = this.#capture(); const controller = new AbortController(); let task: SqlCompletionTask; try { @@ -627,6 +666,7 @@ export function createSqlEditorInternal< ) { if (this.#active === active) { this.#clearCompletionState(); + this.#scheduleCompletionGateClose(capture); } return null; } @@ -704,12 +744,17 @@ export function createSqlEditorInternal< }; readonly update = (update: ViewUpdate): void => { - if ( - !this.#completionPositionIsAllowed( + const completionPositionAllowed = + this.#completionPositionIsAllowed( update.state, update.state.selection.main.head, - ) - ) { + ); + const completionPositionBecameDenied = + this.#completionPositionAllowed && + !completionPositionAllowed; + this.#completionPositionAllowed = completionPositionAllowed; + const completionPositionDenied = !completionPositionAllowed; + if (completionPositionDenied) { this.#clearCompletionState(); } let contextChanged = false; @@ -795,6 +840,12 @@ export function createSqlEditorInternal< this.#clearCompletionState(); } const nextCompletionStatus = completionStatus(update.state); + if ( + completionPositionBecameDenied && + nextCompletionStatus !== null + ) { + this.#scheduleCompletionGateClose(this.#capture()); + } const nextSelectedCompletion = selectedCompletion(update.state); const nextSelectedCompletionIndex = selectedCompletionIndex( update.state, diff --git a/src/vnext/column-catalog-batch-coordinator.ts b/src/vnext/column-catalog-batch-coordinator.ts index bdaec68..38ba9d7 100644 --- a/src/vnext/column-catalog-batch-coordinator.ts +++ b/src/vnext/column-catalog-batch-coordinator.ts @@ -98,28 +98,30 @@ export type SqlColumnCatalogBatchCoordinatorResult = }; interface CoordinatorState { - readonly cache: Map; - readonly capturedProvider: CapturedSqlColumnCatalogProvider; - readonly context: SqlCapturedColumnCatalogProviderContext; - disposed: boolean; - readonly maxCacheEntries: number; - readonly owners: Set; + readonly map: Map; + off: boolean; + readonly id: string; + readonly limit: number; + readonly load: SqlCapturedColumnCatalogProviderContext["loadColumns"]; + readonly pool: Set; + readonly wire: CapturedSqlColumnCatalogProvider; } interface OwnerState { - active: ConsumerState | null; - readonly dialectId: string; - disposed: boolean; - owner: CoordinatorState | null; - observedEpoch: SqlCatalogEpoch | null; + readonly lang: string; + gen: number; + job: ConsumerState | null; + off: boolean; + rev: SqlCatalogEpoch | null; + root: CoordinatorState | null; readonly scope: string; } interface ConsumerState { - readonly controller: AbortController; - readonly owner: OwnerState; - resolve: ((value: SqlColumnCatalogBatchOutcome) => void) | null; - settled: boolean; + readonly abort: AbortController; + done: boolean; + end: ((value: SqlColumnCatalogBatchOutcome) => void) | null; + readonly host: OwnerState; } type ReadyRelation = Extract< @@ -194,10 +196,10 @@ function cacheGet( state: CoordinatorState, key: string, ): ReadyRelation | null { - const value = state.cache.get(key); + const value = state.map.get(key); if (!value) return null; - state.cache.delete(key); - state.cache.set(key, value); + state.map.delete(key); + state.map.set(key, value); return value; } @@ -206,11 +208,11 @@ function cacheSet( key: string, value: ReadyRelation, ): void { - state.cache.delete(key); - state.cache.set(key, value); - while (state.cache.size > state.maxCacheEntries) { - for (const oldest of state.cache.keys()) { - state.cache.delete(oldest); + state.map.delete(key); + state.map.set(key, value); + while (state.map.size > state.limit) { + for (const oldest of state.map.keys()) { + state.map.delete(oldest); break; } } @@ -229,10 +231,13 @@ function settle( consumer: ConsumerState, outcome: SqlColumnCatalogBatchOutcome, ): void { - consumer.settled = true; - consumer.owner.active = null; - const resolve = consumer.resolve; - consumer.resolve = null; + if (consumer.host.job === consumer) { + consumer.host.job = null; + } + if (consumer.done) return; + consumer.done = true; + const resolve = consumer.end; + consumer.end = null; resolve?.(outcome); } @@ -240,8 +245,8 @@ function cancelConsumer( consumer: ConsumerState, outcome: SqlColumnCatalogBatchOutcome, ): void { - if (consumer.settled) return; - consumer.controller.abort(); + if (consumer.done) return; + consumer.abort.abort(); settle(consumer, outcome); } @@ -281,9 +286,9 @@ function startProviderWork( ): void { let providerResult: unknown; try { - providerResult = state.context.loadColumns( + providerResult = state.load( missingRequest, - consumer.controller.signal, + consumer.abort.signal, ); } catch { settle(consumer, unavailable("provider-failed")); @@ -291,17 +296,25 @@ function startProviderWork( } Promise.resolve(providerResult).then( (value) => { - if (consumer.settled) return; - const decoded = decodeSqlColumnCatalogBatchResponse( - state.capturedProvider, - missingRequest, - value, - ); + if (consumer.done) return; + let decoded: ReturnType< + typeof decodeSqlColumnCatalogBatchResponse + >; + try { + decoded = decodeSqlColumnCatalogBatchResponse( + state.wire, + missingRequest, + value, + ); + } catch { + settle(consumer, unavailable("malformed-response")); + return; + } if (decoded.status === "malformed") { settle(consumer, unavailable("malformed-response")); return; } - owner.observedEpoch = decoded.value.epoch; + owner.rev = decoded.value.epoch; for (const relation of decoded.value.relations) { if ( relation.status !== "ready" || @@ -325,7 +338,7 @@ function startProviderWork( settle( consumer, Object.freeze({ - providerId: state.context.id, + providerId: state.id, epoch: decoded.value.epoch, relations, scope: owner.scope, @@ -334,7 +347,7 @@ function startProviderWork( ); }, () => { - if (!consumer.settled) { + if (!consumer.done) { settle(consumer, unavailable("provider-failed")); } }, @@ -345,8 +358,8 @@ function requestColumns( owner: OwnerState, input: unknown, ): SqlColumnCatalogBatchTicket { - const state = owner.owner; - if (!state || state.disposed || owner.disposed) { + const state = owner.root; + if (!state || state.off || owner.off) { return makeSettledTicket(unavailable("disposed")); } const expectedEpoch = dataProperty(input, "expectedEpoch"); @@ -356,9 +369,9 @@ function requestColumns( return makeSettledTicket(unavailable("invalid-request")); } const created = createSqlColumnCatalogBatchRequest({ - dialectId: owner.dialectId, + dialectId: owner.lang, expectedEpoch: expectedEpoch.value === null - ? owner.observedEpoch + ? owner.rev : expectedEpoch.value, relations: relations.value, scope: owner.scope, @@ -368,7 +381,14 @@ function requestColumns( return makeSettledTicket(unavailable("invalid-request")); } const request = created.value; - if (owner.active) cancelConsumer(owner.active, SUPERSEDED); + owner.gen += 1; + const generation = owner.gen; + if (owner.job) cancelConsumer(owner.job, SUPERSEDED); + if (generation !== owner.gen) { + return makeSettledTicket( + owner.off ? unavailable("disposed") : SUPERSEDED, + ); + } const cached = new Map(); const missing: SqlColumnCatalogRelationReference[] = []; @@ -387,7 +407,7 @@ function requestColumns( return makeSettledTicket( Object.freeze({ epoch: request.expectedEpoch, - providerId: state.context.id, + providerId: state.id, relations: combineRelations(cached, []), scope: owner.scope, status: "usable", @@ -405,12 +425,12 @@ function requestColumns( resolveResult = resolve; }); const consumer: ConsumerState = { - controller: new AbortController(), - owner, - resolve: resolveResult, - settled: false, + abort: new AbortController(), + done: false, + end: resolveResult, + host: owner, }; - owner.active = consumer; + owner.job = consumer; const ticket = Object.freeze({ cancel: (): void => cancelConsumer(consumer, CANCELLED), result, @@ -426,20 +446,21 @@ function requestColumns( } function disposeOwner(owner: OwnerState): void { - if (owner.disposed) return; - owner.disposed = true; - if (owner.active) { - cancelConsumer(owner.active, unavailable("disposed")); + if (owner.off) return; + owner.off = true; + owner.gen += 1; + if (owner.job) { + cancelConsumer(owner.job, unavailable("disposed")); } - owner.owner?.owners.delete(owner); - owner.owner = null; + owner.root?.pool.delete(owner); + owner.root = null; } function prepareOwner( state: CoordinatorState, input: unknown, ): SqlColumnCatalogBatchOwnerResult { - if (state.disposed) { + if (state.off) { return Object.freeze({ reason: "disposed", status: "unavailable" }); } const scopeProperty = dataProperty(input, "scope"); @@ -459,14 +480,15 @@ function prepareOwner( }); } const owner: OwnerState = { - active: null, - dialectId, - disposed: false, - owner: state, - observedEpoch: null, + gen: 0, + job: null, + lang: dialectId, + off: false, + rev: null, + root: state, scope, }; - state.owners.add(owner); + state.pool.add(owner); return Object.freeze({ owner: Object.freeze({ dispose: (): void => disposeOwner(owner), @@ -478,10 +500,10 @@ function prepareOwner( } function disposeCoordinator(state: CoordinatorState): void { - if (state.disposed) return; - state.disposed = true; - state.cache.clear(); - for (const owner of state.owners) disposeOwner(owner); + if (state.off) return; + state.off = true; + state.map.clear(); + for (const owner of state.pool) disposeOwner(owner); } export function createSqlColumnCatalogBatchCoordinator( @@ -524,19 +546,20 @@ export function createSqlColumnCatalogBatchCoordinator( }); } const state: CoordinatorState = { - cache: new Map(), - capturedProvider: captured.value, - context, - disposed: false, - maxCacheEntries: maximum, - owners: new Set(), + id: context.id, + limit: maximum, + load: context.loadColumns, + map: new Map(), + off: false, + pool: new Set(), + wire: captured.value, }; return Object.freeze({ coordinator: Object.freeze({ dispose: (): void => disposeCoordinator(state), prepareOwner: (options: SqlColumnCatalogOwnerOptions) => prepareOwner(state, options), - providerId: context.id, + providerId: state.id, }), status: "created", }); diff --git a/src/vnext/column-catalog-boundary.ts b/src/vnext/column-catalog-boundary.ts index 13a03fe..3c1f573 100644 --- a/src/vnext/column-catalog-boundary.ts +++ b/src/vnext/column-catalog-boundary.ts @@ -7,9 +7,10 @@ import type { SqlColumnCatalogRelationResult, SqlColumnCatalogResolvedColumn, } from "./column-catalog-types.js"; -import type { - SqlIdentifierComponent, - SqlIdentifierPath, +import { + isDataArray, + type SqlIdentifierComponent, + type SqlIdentifierPath, } from "./types.js"; export const MAX_COLUMN_PROVIDER_ID_LENGTH = 256; @@ -121,7 +122,7 @@ function readRecord( value: unknown, allowedKeys: ReadonlySet, ): DataRecord | null { - if (value === null || typeof value !== "object" || Array.isArray(value)) { + if (value === null || typeof value !== "object") { return null; } let keys: readonly PropertyKey[]; @@ -210,7 +211,7 @@ function readArrayElement( } function readArrayLength(value: unknown, maximum: number): number | null { - if (!Array.isArray(value)) return null; + if (!isDataArray(value)) return null; let descriptor: PropertyDescriptor | undefined; try { descriptor = Object.getOwnPropertyDescriptor(value, "length"); @@ -497,16 +498,11 @@ function decodeColumn( }); } -function columnKey(column: SqlColumnCatalogResolvedColumn): string { - return [ - column.columnEntityId, - column.ordinal, - column.identifier.quoted ? "q" : "u", - column.identifier.value, - column.insertText, - column.dataType ?? "", - column.detail ?? "", - ].join("\u0000"); +function sameColumn( + left: SqlColumnCatalogResolvedColumn, + right: SqlColumnCatalogResolvedColumn, +): boolean { + return JSON.stringify(left) === JSON.stringify(right); } function decodeReadyRelation( @@ -526,7 +522,7 @@ function decodeReadyRelation( if ( (coverage !== "complete" && coverage !== "partial") || columnCount === null || - !Array.isArray(columnsValue) + !isDataArray(columnsValue) ) { return malformed("invalid-shape"); } @@ -543,7 +539,7 @@ function decodeReadyRelation( ); if (!column) return malformed("invalid-shape"); const previous = byId.get(column.columnEntityId); - if (previous && columnKey(previous) !== columnKey(column)) { + if (previous && !sameColumn(previous, column)) { return malformed("duplicate-column-entity-id"); } byId.set(column.columnEntityId, column); @@ -592,7 +588,10 @@ function decodeRelation( required(record, "relationEntityId"), MAX_COLUMN_ENTITY_ID_LENGTH, ); - if (relationEntityId === null) return malformed("invalid-shape"); + if ( + relationEntityId === null || + record.fields.size !== 5 + ) return malformed("invalid-shape"); return decodeReadyRelation( record, providerId, @@ -603,6 +602,7 @@ function decodeRelation( ); } if (status === "loading") { + if (record.fields.size !== 2) return malformed("invalid-shape"); return accepted(Object.freeze({ requestKey, status, @@ -612,6 +612,7 @@ function decodeRelation( const retry = required(record, "retry"); if ( status !== "failed" || + record.fields.size !== 4 || !isFailureCode(code) || !isRetryPolicy(retry) ) { diff --git a/src/vnext/column-completion.ts b/src/vnext/column-completion.ts index 228f8f2..8ea49e9 100644 --- a/src/vnext/column-completion.ts +++ b/src/vnext/column-completion.ts @@ -1,6 +1,9 @@ import type { SqlColumnCatalogBatchOutcome, } from "./column-catalog-batch-coordinator.js"; +import { + MAX_COLUMN_BATCH_RELATIONS, +} from "./column-catalog-boundary.js"; import type { SqlColumnCatalogRelationReference, SqlColumnCatalogRelationResult, @@ -28,6 +31,7 @@ type ReadyColumnSite = Extract< >; export interface SqlPreparedColumnCatalogRelations { + readonly coverage: "complete" | "partial"; readonly references: readonly SqlColumnCatalogRelationReference[]; readonly relationsByRequestKey: ReadonlyMap< string, @@ -89,6 +93,7 @@ export function prepareSqlColumnCatalogRelations( string, SqlColumnQueryRelation >(); + let coverage: "complete" | "partial" = "complete"; for (let index = 0; index < site.relations.length; index += 1) { const relation = site.relations[index]; if ( @@ -101,6 +106,10 @@ export function prepareSqlColumnCatalogRelations( ) { continue; } + if (references.length === MAX_COLUMN_BATCH_RELATIONS) { + coverage = "partial"; + break; + } const requestKey = `binding:${index}`; references.push(Object.freeze({ path: relation.path, @@ -109,6 +118,7 @@ export function prepareSqlColumnCatalogRelations( relationsByRequestKey.set(requestKey, relation); } return Object.freeze({ + coverage, references: Object.freeze(references), relationsByRequestKey, }); @@ -222,12 +232,12 @@ export function composeSqlColumnCompletion( } const items: SqlCompletionItem[] = []; const issues: SqlCompletionIssue[] = []; - if (input.site.coverage === "partial") { - issues.push(issue("query-binding-partial")); - } + let hasPartial = + input.site.coverage === "partial" || + input.prepared.coverage === "partial"; + if (hasPartial) issues.push(issue("query-binding-partial")); let hasLoading = false; let hasFailure = false; - let hasPartial = input.site.coverage === "partial"; const seen = new Set(); const failures: SqlColumnCatalogFailure[] = []; for (const result of input.outcome.relations) { diff --git a/src/vnext/column-query-site.ts b/src/vnext/column-query-site.ts index c867225..c66a0a5 100644 --- a/src/vnext/column-query-site.ts +++ b/src/vnext/column-query-site.ts @@ -244,8 +244,7 @@ function parseNamedRelation( if ( isIdentifier(aliasToken) && aliasToken?.depth === depth && - (maybeAs === "as" || - aliasWord === null || + (aliasWord === null || !RELATION_END_WORDS.has(aliasWord)) ) { const aliasPath = decodePath( @@ -260,7 +259,10 @@ function parseNamedRelation( } } return { - issue: null, + issue: + maybeAs === "as" && alias === null + ? "incomplete-relation" + : null, next: end, relation: Object.freeze({ alias, @@ -382,7 +384,10 @@ function cursorTokenReason( { status: "inactive" } >["reason"] | null { const token = tokens.find((candidate) => - candidate.from <= position && position < candidate.to + candidate.from <= position && + (position < candidate.to || + (candidate.kind === "line-comment" && + position === candidate.to)) ); switch (token?.kind) { case "barrier": @@ -397,66 +402,24 @@ function cursorTokenReason( } } -export function recognizeSqlColumnQuerySite( +function collectRelations( source: SqlSourceSnapshot, - slot: SqlStatementSlot, - position: number, dialect: SqlRelationDialectRuntime, -): SqlColumnQuerySiteResult { - if ( - !isSqlSourceSnapshot(source) || - !isSqlStatementSlotSnapshot(slot) || - !isSqlRelationDialectRuntime(dialect) || - !Number.isSafeInteger(position) || - position < 0 || - position > source.analysisText.length - ) { - return unavailable("ambiguous-query-site"); - } - if (slot.boundaryQuality === "opaque") { - return unavailable("opaque-statement"); - } - const tokens = tokenize(source, slot, dialect); - if (tokens === null) return unavailable("resource-limit"); - const cursorReason = cursorTokenReason(tokens, position); - if (cursorReason !== null) return inactive(cursorReason); - - let cursorDepth = 0; - for (const token of tokens) { - if (token.from >= position) break; - cursorDepth = token.depth; - if (punctuation(source, token, "(")) cursorDepth += 1; - } - let selectIndex = -1; - for (let index = 0; index < tokens.length; index += 1) { - const token = tokens[index]!; - if ( - token.from <= position && - token.depth <= cursorDepth && - word(source, token) === "select" - ) { - selectIndex = index; - } - } - if (selectIndex < 0) return inactive("not-select-query"); - const selectDepth = tokens[selectIndex]!.depth; + tokens: readonly Token[], + selectIndex: number, + visibilityPosition: number, + relations: SqlColumnQueryRelation[], + issues: Set, +): boolean { + const selectDepth = tokens[selectIndex]?.depth; + if (selectDepth === undefined) return false; const clause = clauseAt( source, tokens, selectIndex, selectDepth, - position, + visibilityPosition, ); - if (clause === "from" || clause === "limit") { - return inactive("not-column-position"); - } - const path = typedPath(source, tokens, position, dialect); - if (path === null) { - return inactive("not-column-position"); - } - - const relations: SqlColumnQueryRelation[] = []; - const issues = new Set(); let commaStartsRelation = false; let inFrom = false; for ( @@ -467,7 +430,7 @@ export function recognizeSqlColumnQuerySite( const token = tokens[index]!; if ( clause === "join-condition" && - token.from >= position + token.from >= visibilityPosition ) { break; } @@ -518,13 +481,173 @@ export function recognizeSqlColumnQuerySite( if (parsed.issue !== null) issues.add(parsed.issue); if (parsed.relation !== null) { relations.push(parsed.relation); - if (relations.length > MAX_COLUMN_QUERY_RELATIONS) { - return unavailable("resource-limit"); - } + if (relations.length > MAX_COLUMN_QUERY_RELATIONS) return false; } commaStartsRelation = true; index = Math.max(index, parsed.next - 1); } + return true; +} + +function openingBefore( + source: SqlSourceSnapshot, + tokens: readonly Token[], + beforeIndex: number, + depth: number, +): number { + for (let index = beforeIndex - 1; index >= 0; index -= 1) { + const token = tokens[index]!; + if ( + token.depth === depth && + punctuation(source, token, "(") + ) { + return token.from; + } + } + return -1; +} + +function correlatedParentSelectIndex( + source: SqlSourceSnapshot, + tokens: readonly Token[], + childIndex: number, +): number { + const child = tokens[childIndex]; + if (!child || child.depth === 0) return -1; + for (let depth = child.depth - 1; depth >= 0; depth -= 1) { + const lowerBound = depth === 0 + ? -1 + : openingBefore(source, tokens, childIndex, depth - 1); + for (let index = childIndex - 1; index >= 0; index -= 1) { + const token = tokens[index]!; + if (token.from <= lowerBound) break; + if (token.depth === depth && word(source, token) === "select") { + const opening = openingBefore( + source, + tokens, + childIndex, + depth, + ); + return opening > token.from && + clauseAt( + source, + tokens, + index, + depth, + opening, + ) !== "from" + ? index + : -1; + } + } + } + return -1; +} + +export function recognizeSqlColumnQuerySite( + source: SqlSourceSnapshot, + slot: SqlStatementSlot, + position: number, + dialect: SqlRelationDialectRuntime, +): SqlColumnQuerySiteResult { + if ( + !isSqlSourceSnapshot(source) || + !isSqlStatementSlotSnapshot(slot) || + !isSqlRelationDialectRuntime(dialect) || + !Number.isSafeInteger(position) || + position < 0 || + position > source.analysisText.length + ) { + return unavailable("ambiguous-query-site"); + } + if (slot.boundaryQuality === "opaque") { + return unavailable("opaque-statement"); + } + if ( + position < slot.source.from || + position > slot.source.to + ) { + return inactive("not-column-position"); + } + const tokens = tokenize(source, slot, dialect); + if (tokens === null) return unavailable("resource-limit"); + const cursorReason = cursorTokenReason(tokens, position); + if (cursorReason !== null) return inactive(cursorReason); + + let cursorDepth = 0; + for (const token of tokens) { + if (token.from >= position) break; + cursorDepth = token.depth; + if (punctuation(source, token, "(")) cursorDepth += 1; + } + let selectIndex = -1; + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]!; + if ( + token.from <= position && + token.depth <= cursorDepth && + word(source, token) === "select" + ) { + selectIndex = index; + } + } + if (selectIndex < 0) return inactive("not-select-query"); + const selectDepth = tokens[selectIndex]!.depth; + const clause = clauseAt( + source, + tokens, + selectIndex, + selectDepth, + position, + ); + if (clause === "from" || clause === "limit") { + return inactive("not-column-position"); + } + const path = typedPath(source, tokens, position, dialect); + if (path === null) { + return inactive("not-column-position"); + } + + const relations: SqlColumnQueryRelation[] = []; + const issues = new Set(); + if ( + !collectRelations( + source, + dialect, + tokens, + selectIndex, + position, + relations, + issues, + ) + ) { + return unavailable("resource-limit"); + } + let childIndex = selectIndex; + for (;;) { + const parentIndex = correlatedParentSelectIndex( + source, + tokens, + childIndex, + ); + if (parentIndex < 0) break; + const childPosition = tokens[childIndex]?.from; + if ( + childPosition === undefined || + !collectRelations( + source, + dialect, + tokens, + parentIndex, + childPosition, + relations, + issues, + ) + ) { + return unavailable("resource-limit"); + } + childIndex = parentIndex; + } const issueList = Object.freeze(Array.from(issues).sort()); return Object.freeze({ coverage: issueList.length === 0 ? "complete" : "partial", diff --git a/src/vnext/namespace-catalog-boundary.ts b/src/vnext/namespace-catalog-boundary.ts index 78c8113..efe718b 100644 --- a/src/vnext/namespace-catalog-boundary.ts +++ b/src/vnext/namespace-catalog-boundary.ts @@ -8,9 +8,10 @@ import type { SqlNamespaceContainerRole, SqlNamespacePathComponent, } from "./namespace-catalog-types.js"; -import type { - SqlIdentifierComponent, - SqlIdentifierPath, +import { + isDataArray, + type SqlIdentifierComponent, + type SqlIdentifierPath, } from "./types.js"; export const MAX_NAMESPACE_PROVIDER_ID_LENGTH = 256; @@ -84,7 +85,10 @@ function record( value: unknown, allowed: ReadonlySet, ): DataRecord | null { - if (value === null || typeof value !== "object") return null; + if ( + value === null || + typeof value !== "object" + ) return null; let keys: readonly PropertyKey[]; try { keys = Reflect.ownKeys(value); @@ -131,7 +135,7 @@ function arrayLength( value: unknown, maximum: number, ): number | null { - if (!Array.isArray(value)) return null; + if (!isDataArray(value)) return null; let descriptor: PropertyDescriptor | undefined; try { descriptor = Object.getOwnPropertyDescriptor(value, "length"); @@ -153,7 +157,7 @@ function arrayElement( value: unknown, index: number, ): unknown | null { - if (!Array.isArray(value)) return null; + if (!isDataArray(value)) return null; let descriptor: PropertyDescriptor | undefined; try { descriptor = Object.getOwnPropertyDescriptor(value, String(index)); @@ -167,6 +171,7 @@ function arrayElement( function identifier( value: unknown, + allowEmpty = false, ): SqlIdentifierComponent | null { const source = record(value, new Set(["quoted", "value"])); if (!source) return null; @@ -174,7 +179,7 @@ function identifier( const text = boundedString( required(source, "value"), MAX_NAMESPACE_IDENTIFIER_LENGTH, - true, + allowEmpty, ); return typeof quoted === "boolean" && text !== null ? Object.freeze({ quoted, value: text }) @@ -275,10 +280,22 @@ function compareText(left: string, right: string): number { return left < right ? -1 : left > right ? 1 : 0; } -function pathKey(path: SqlCanonicalNamespacePath): string { - return path.map((component) => - `${component.role}:${component.quoted ? "q" : "u"}:${component.value}` - ).join("\u0000"); +function comparePath( + left: SqlCanonicalNamespacePath, + right: SqlCanonicalNamespacePath, +): number { + const length = Math.min(left.length, right.length); + for (let index = 0; index < length; index += 1) { + const leftComponent = left[index]; + const rightComponent = right[index]; + if (!leftComponent || !rightComponent) continue; + const compared = + compareText(leftComponent.role, rightComponent.role) || + Number(rightComponent.quoted) - Number(leftComponent.quoted) || + compareText(leftComponent.value, rightComponent.value); + if (compared !== 0) return compared; + } + return left.length - right.length; } function sameContainer( @@ -289,7 +306,7 @@ function sameContainer( left.detail === right.detail && left.insertText === right.insertText && left.matchQuality === right.matchQuality && - pathKey(left.canonicalPath) === pathKey(right.canonicalPath); + comparePath(left.canonicalPath, right.canonicalPath) === 0; } export function captureSqlNamespaceCatalogProvider( @@ -359,7 +376,7 @@ export function createSqlNamespaceCatalogSearchRequest( ? null : epoch(expectedValue); const limit = required(source, "limit"); - const prefix = identifier(required(source, "prefix")); + const prefix = identifier(required(source, "prefix"), true); const qualifier = identifierPath( required(source, "qualifier"), MAX_NAMESPACE_PATH_COMPONENTS, @@ -546,7 +563,7 @@ export function decodeSqlNamespaceCatalogSearchResponse( ? -1 : 1) || left.canonicalPath.length - right.canonicalPath.length || - compareText(pathKey(left.canonicalPath), pathKey(right.canonicalPath)) || + comparePath(left.canonicalPath, right.canonicalPath) || compareText(left.containerEntityId, right.containerEntityId) ); return accepted(Object.freeze({ diff --git a/src/vnext/namespace-catalog-coordinator.ts b/src/vnext/namespace-catalog-coordinator.ts index 7b7d71d..ca9e731 100644 --- a/src/vnext/namespace-catalog-coordinator.ts +++ b/src/vnext/namespace-catalog-coordinator.ts @@ -99,30 +99,32 @@ export type SqlNamespaceCatalogCoordinatorResult = }; interface CoordinatorState { - readonly cache: Map; - readonly capturedProvider: CapturedSqlNamespaceCatalogProvider; - readonly context: SqlCapturedNamespaceCatalogProviderContext; - disposed: boolean; - readonly maxCacheEntries: number; - readonly owners: Set; + readonly map: Map; + off: boolean; + readonly id: string; + readonly limit: number; + readonly load: SqlCapturedNamespaceCatalogProviderContext["search"]; + readonly pool: Set; + readonly wire: CapturedSqlNamespaceCatalogProvider; } interface OwnerState { - active: ConsumerState | null; - readonly dialectId: string; - disposed: boolean; - observedEpoch: SqlCatalogEpoch | null; - owner: CoordinatorState | null; + job: ConsumerState | null; + readonly lang: string; + gen: number; + off: boolean; + rev: SqlCatalogEpoch | null; + root: CoordinatorState | null; readonly scope: string; } interface ConsumerState { - readonly controller: AbortController; - readonly owner: OwnerState; - resolve: + readonly abort: AbortController; + done: boolean; + end: | ((value: SqlNamespaceCatalogSearchOutcome) => void) | null; - settled: boolean; + readonly host: OwnerState; } const CANCELLED: SqlNamespaceCatalogSearchOutcome = @@ -197,10 +199,10 @@ function cacheGet( state: CoordinatorState, key: string, ): SqlNamespaceCatalogSearchResponse | null { - const value = state.cache.get(key); + const value = state.map.get(key); if (!value) return null; - state.cache.delete(key); - state.cache.set(key, value); + state.map.delete(key); + state.map.set(key, value); return value; } @@ -209,12 +211,13 @@ function cacheSet( key: string, value: SqlNamespaceCatalogSearchResponse, ): void { - state.cache.delete(key); - state.cache.set(key, value); - while (state.cache.size > state.maxCacheEntries) { - const oldest = state.cache.keys().next().value; - if (typeof oldest !== "string") break; - state.cache.delete(oldest); + state.map.delete(key); + state.map.set(key, value); + while (state.map.size > state.limit) { + for (const oldest of state.map.keys()) { + state.map.delete(oldest); + break; + } } } @@ -222,13 +225,13 @@ function settle( consumer: ConsumerState, outcome: SqlNamespaceCatalogSearchOutcome, ): void { - if (consumer.settled) return; - consumer.settled = true; - if (consumer.owner.active === consumer) { - consumer.owner.active = null; + if (consumer.host.job === consumer) { + consumer.host.job = null; } - const resolve = consumer.resolve; - consumer.resolve = null; + if (consumer.done) return; + consumer.done = true; + const resolve = consumer.end; + consumer.end = null; resolve?.(outcome); } @@ -236,8 +239,8 @@ function cancel( consumer: ConsumerState, outcome: SqlNamespaceCatalogSearchOutcome, ): void { - if (consumer.settled) return; - consumer.controller.abort(); + if (consumer.done) return; + consumer.abort.abort(); settle(consumer, outcome); } @@ -258,9 +261,9 @@ function providerWork( ): void { let pending: unknown; try { - pending = state.context.search( + pending = state.load( request, - consumer.controller.signal, + consumer.abort.signal, ); } catch { settle(consumer, unavailable("provider-failed")); @@ -269,16 +272,16 @@ function providerWork( Promise.resolve(pending).then( (value) => { if ( - consumer.settled || - consumer.controller.signal.aborted || - state.disposed || - owner.disposed || - owner.owner !== state + consumer.done || + consumer.abort.signal.aborted || + state.off || + owner.off || + owner.root !== state ) { return; } const decoded = decodeSqlNamespaceCatalogSearchResponse( - state.capturedProvider, + state.wire, request, value, ); @@ -286,7 +289,7 @@ function providerWork( settle(consumer, unavailable("malformed-response")); return; } - owner.observedEpoch = decoded.value.epoch; + owner.rev = decoded.value.epoch; if ( decoded.value.status === "ready" && decoded.value.coverage === "complete" @@ -298,14 +301,14 @@ function providerWork( ); } settle(consumer, Object.freeze({ - providerId: state.context.id, + providerId: state.id, response: decoded.value, scope: owner.scope, status: "usable", })); }, () => { - if (!consumer.settled) { + if (!consumer.done) { settle(consumer, unavailable("provider-failed")); } }, @@ -316,8 +319,8 @@ function request( owner: OwnerState, input: unknown, ): SqlNamespaceCatalogSearchTicket { - const state = owner.owner; - if (!state || state.disposed || owner.disposed) { + const state = owner.root; + if (!state || state.off || owner.off) { return settledTicket(unavailable("disposed")); } const expected = property(input, "expectedEpoch"); @@ -329,9 +332,9 @@ function request( return settledTicket(unavailable("invalid-request")); } const created = createSqlNamespaceCatalogSearchRequest({ - dialectId: owner.dialectId, + dialectId: owner.lang, expectedEpoch: expected.value === null - ? owner.observedEpoch + ? owner.rev : expected.value, limit: limit.value, prefix: prefix.value, @@ -342,7 +345,14 @@ function request( if (created.status === "malformed") { return settledTicket(unavailable("invalid-request")); } - if (owner.active) cancel(owner.active, SUPERSEDED); + owner.gen += 1; + const generation = owner.gen; + if (owner.job) cancel(owner.job, SUPERSEDED); + if (generation !== owner.gen) { + return settledTicket( + owner.off ? unavailable("disposed") : SUPERSEDED, + ); + } const normalized = created.value; const key = normalized.expectedEpoch === null ? null @@ -350,29 +360,27 @@ function request( const cached = key === null ? null : cacheGet(state, key); if (cached) { return settledTicket(Object.freeze({ - providerId: state.context.id, + providerId: state.id, response: cached, scope: owner.scope, status: "usable", })); } - const resolver: { - value: - | ((value: SqlNamespaceCatalogSearchOutcome) => void) - | null; - } = { value: null }; + let resolveResult: ( + value: SqlNamespaceCatalogSearchOutcome + ) => void = (): void => {}; const result = new Promise( (resolve) => { - resolver.value = resolve; + resolveResult = resolve; }, ); const consumer: ConsumerState = { - controller: new AbortController(), - owner, - resolve: resolver.value, - settled: false, + abort: new AbortController(), + done: false, + end: resolveResult, + host: owner, }; - owner.active = consumer; + owner.job = consumer; const ticket = Object.freeze({ cancel: (): void => cancel(consumer, CANCELLED), result, @@ -382,18 +390,19 @@ function request( } function disposeOwner(owner: OwnerState): void { - if (owner.disposed) return; - owner.disposed = true; - if (owner.active) cancel(owner.active, unavailable("disposed")); - owner.owner?.owners.delete(owner); - owner.owner = null; + if (owner.off) return; + owner.off = true; + owner.gen += 1; + if (owner.job) cancel(owner.job, unavailable("disposed")); + owner.root?.pool.delete(owner); + owner.root = null; } function prepareOwner( state: CoordinatorState, value: unknown, ): SqlNamespaceCatalogOwnerResult { - if (state.disposed) { + if (state.off) { return Object.freeze({ reason: "disposed", status: "unavailable" }); } const dialectId = boundedString( @@ -411,14 +420,15 @@ function prepareOwner( }); } const owner: OwnerState = { - active: null, - dialectId, - disposed: false, - observedEpoch: null, - owner: state, + job: null, + gen: 0, + lang: dialectId, + off: false, + rev: null, + root: state, scope, }; - state.owners.add(owner); + state.pool.add(owner); return Object.freeze({ owner: Object.freeze({ dispose: (): void => disposeOwner(owner), @@ -430,10 +440,10 @@ function prepareOwner( } function dispose(state: CoordinatorState): void { - if (state.disposed) return; - state.disposed = true; - state.cache.clear(); - for (const owner of state.owners) disposeOwner(owner); + if (state.off) return; + state.off = true; + state.map.clear(); + for (const owner of state.pool) disposeOwner(owner); } export function createSqlNamespaceCatalogCoordinator( @@ -471,19 +481,20 @@ export function createSqlNamespaceCatalogCoordinator( }); } const state: CoordinatorState = { - cache: new Map(), - capturedProvider: captured.value, - context, - disposed: false, - maxCacheEntries: maximum, - owners: new Set(), + id: context.id, + limit: maximum, + load: context.search, + map: new Map(), + off: false, + pool: new Set(), + wire: captured.value, }; return Object.freeze({ coordinator: Object.freeze({ dispose: (): void => dispose(state), prepareOwner: (options: SqlNamespaceCatalogOwnerOptions) => prepareOwner(state, options), - providerId: context.id, + providerId: state.id, }), status: "created", }); diff --git a/src/vnext/node-sql-parser-query-bindings.ts b/src/vnext/node-sql-parser-query-bindings.ts index e106c8e..83a9776 100644 --- a/src/vnext/node-sql-parser-query-bindings.ts +++ b/src/vnext/node-sql-parser-query-bindings.ts @@ -59,7 +59,9 @@ interface SelectSkeleton { } interface AstSelect { - readonly children: AstSelect[]; + readonly cteChildren: AstSelect[]; + readonly derivedChildren: AstSelect[]; + readonly kind: "compound" | "select"; readonly node: object; skeleton: SelectSkeleton | null; } @@ -160,7 +162,11 @@ function arrayProperty( if (property.value === null) { return []; } - if (!Array.isArray(property.value)) { + try { + if (!Array.isArray(property.value)) { + return null; + } + } catch { return null; } const length = ownProperty(property.value, "length"); @@ -259,9 +265,27 @@ function selectSkeletons( break; } } + let from = 0; + if (token.depth > 0) { + for (let cursor = index - 1; cursor >= 0; cursor -= 1) { + const candidate = tokens[cursor]; + if ( + candidate?.text !== "(" || + candidate.depth !== token.depth - 1 + ) { + continue; + } + const close = matchingClose(tokens, cursor, candidate.depth); + if (close !== null && phaseValue(tokens, close).from < token.from) { + continue; + } + from = candidate.to; + break; + } + } skeletons.push(Object.freeze({ depth: token.depth, - from: token.from, + from, selectToken: index, to, })); @@ -276,11 +300,19 @@ function astSelectNode(value: unknown): object | null { const type = stringProperty(value, "type"); return type !== null && (type.toLowerCase() === "select" || - type.toLowerCase() === "union") + type.toLowerCase() === "union" || + type.toLowerCase() === "intersect" || + type.toLowerCase() === "except") ? value : null; } +function astSelectKind(value: object): "compound" | "select" { + return stringProperty(value, "type")?.toLowerCase() === "select" + ? "select" + : "compound"; +} + function derivedAst(value: object): object | null { const expression = objectProperty(value, "expr"); if (expression === null) { @@ -303,7 +335,9 @@ function buildAstTree( seen.add(node); count += 1; const result: AstSelect = { - children: [], + cteChildren: [], + derivedChildren: [], + kind: astSelectKind(node), node, skeleton: null, }; @@ -323,7 +357,7 @@ function buildAstTree( if (child === null) { return null; } - result.children.push(child); + result.cteChildren.push(child); } textual.push(result); const fromItems = arrayProperty(node, "from"); @@ -340,7 +374,7 @@ function buildAstTree( if (child === null) { return null; } - result.children.push(child); + result.derivedChildren.push(child); } } return result; @@ -389,7 +423,7 @@ function emitBlocks( ), skeleton, }); - for (const child of ast.children) { + for (const child of [...ast.cteChildren, ...ast.derivedChildren]) { emit(child, id); } } @@ -579,25 +613,94 @@ function pathFromAst( return null; } const lexicalIdentifiers = site.lexicalIdentifiers; - const allQuoted = + const singleQuotedBigQueryPath = grammar === "bigquery" && lexicalIdentifiers.length === 1 && lexicalIdentifiers[0]?.quoted === true; - return Object.freeze(values.map((value, index) => - Object.freeze({ - quoted: allQuoted || lexicalIdentifiers[index]?.quoted === true, - value, + if (singleQuotedBigQueryPath) { + if (lexicalIdentifiers[0]?.value !== values.join(".")) { + return null; + } + return Object.freeze(values.map((value) => + Object.freeze({ quoted: true, value }) + )); + } + if ( + values.length !== lexicalIdentifiers.length || + values.some((value, index) => { + const lexical = lexicalIdentifiers[index]; + return lexical === undefined || + !identifierMatchesAst(value, lexical); }) - )); + ) { + return null; + } + return Object.freeze([...lexicalIdentifiers]); } -function cteDeclarations(root: object, tokens: readonly Token[]) { - const declarations = new Map(); - const withItems = arrayProperty(root, "with"); +interface CteDeclaration { + readonly name: SqlIdentifierComponent; + readonly range: SqlTextRange; +} + +interface IdentifierToken { + readonly identifier: SqlIdentifierComponent; + readonly token: Token; +} + +function identifierMatchesAst( + astValue: string, + lexical: SqlIdentifierComponent, +): boolean { + return lexical.quoted + ? astValue === lexical.value + : astValue.toLowerCase() === lexical.value.toLowerCase(); +} + +function identifierKey(identifier: SqlIdentifierComponent): string { + return identifier.quoted + ? `quoted:${identifier.value}` + : `unquoted:${identifier.value.toLowerCase()}`; +} + +function findIdentifierToken( + tokens: readonly Token[], + from: number, + to: number, + depth: number, + astName: string, +): IdentifierToken | null { + for (const token of tokens) { + const identifier = tokenIdentifier(token); + if ( + token.from >= from && + token.from < to && + token.depth === depth && + identifier !== null && + identifierMatchesAst(astName, identifier) + ) { + return { identifier, token }; + } + } + return null; +} + +function ownCteDeclarations( + ast: AstSelect, + tokens: readonly Token[], +): readonly CteDeclaration[] | null { + const withItems = arrayProperty(ast.node, "with"); if (withItems === null) { return null; } - for (const item of withItems) { + if (withItems.length !== ast.cteChildren.length) { + return null; + } + const declarations: CteDeclaration[] = []; + let searchFrom = assignedSkeleton(ast).from; + for (let index = 0; index < withItems.length; index += 1) { + const item = withItems[index]; + const child = phaseValue(ast.cteChildren, index); if (item === null || typeof item !== "object") { return null; } @@ -608,19 +711,59 @@ function cteDeclarations(root: object, tokens: readonly Token[]) { if (name === null) { return null; } - const token = tokens.find((candidate) => - candidate.kind === "word" && - candidate.text.toLowerCase() === name.toLowerCase() + const declaration = findIdentifierToken( + tokens, + searchFrom, + assignedSkeleton(child).from, + assignedSkeleton(ast).depth, + name, ); - if (token === undefined) { + if (declaration === null) { return null; } - declarations.set( - name.toLowerCase(), - Object.freeze({ from: token.from, to: token.to }), - ); + declarations.push(Object.freeze({ + name: declaration.identifier, + range: Object.freeze({ + from: declaration.token.from, + to: declaration.token.to, + }), + })); + searchFrom = assignedSkeleton(child).to; } - return declarations; + return Object.freeze(declarations); +} + +function visibleCteDeclarations( + root: AstSelect, + tokens: readonly Token[], +): ReadonlyMap> | null { + const result = new Map>(); + function visit( + ast: AstSelect, + inherited: ReadonlyMap, + ): boolean { + const own = ownCteDeclarations(ast, tokens); + if (own === null) { + return false; + } + const visible = new Map(inherited); + for (let index = 0; index < ast.cteChildren.length; index += 1) { + const child = phaseValue(ast.cteChildren, index); + if (!visit(child, visible)) { + return false; + } + const declaration = phaseValue(own, index); + visible.set(identifierKey(declaration.name), declaration); + } + result.set(ast.node, visible); + for (const child of ast.derivedChildren) { + if (!visit(child, visible)) { + return false; + } + } + return true; + } + return visit(root, new Map()) ? result : null; } function addIssue( @@ -702,37 +845,59 @@ function clauseRegions( function joinRegions( tokens: readonly Token[], block: BlockBuild, + sites: readonly RelationSite[], scopesAfterRelations: readonly number[], ): SqlVisibilityRegion[] { const result: SqlVisibilityRegion[] = []; - let relationIndex = -1; - for ( - let index = block.skeleton.selectToken; - index < tokens.length; - index += 1 - ) { - const token = tokens[index]; - if (!token || token.from >= block.skeleton.to) { - break; - } - if (token.depth !== block.skeleton.depth) { + for (let relationIndex = 1; relationIndex < sites.length; relationIndex += 1) { + const site = phaseValue(sites, relationIndex); + const scope = scopesAfterRelations[relationIndex]; + if (scope === undefined) { continue; } - if (word(token, "from") || word(token, "join") || token.text === ",") { - relationIndex += 1; + let conditionIndex = -1; + for (let index = 0; index < tokens.length; index += 1) { + const candidate = phaseValue(tokens, index); + if ( + candidate.from < site.to || + candidate.depth !== block.skeleton.depth + ) { + continue; + } + if (candidate.from >= block.skeleton.to) { + break; + } + if (word(candidate, "on") || word(candidate, "using")) { + conditionIndex = index; + break; + } + if ( + word(candidate, "join") || + candidate.text === "," || + word(candidate, "where") || + word(candidate, "group") || + word(candidate, "having") || + word(candidate, "qualify") || + word(candidate, "order") || + word(candidate, "limit") + ) { + break; + } } - if (!word(token, "on")) { + const token = tokens[conditionIndex]; + if (token === undefined) { continue; } let to = block.skeleton.to; - for (let cursor = index + 1; cursor < tokens.length; cursor += 1) { - const next = tokens[cursor]; - if (!next || next.from >= block.skeleton.to) { + for (let cursor = conditionIndex + 1; cursor < tokens.length; cursor += 1) { + const next = phaseValue(tokens, cursor); + if (next.from >= block.skeleton.to) { break; } if ( next.depth === block.skeleton.depth && (word(next, "join") || + next.text === "," || word(next, "where") || word(next, "group") || word(next, "having") || @@ -744,8 +909,7 @@ function joinRegions( break; } } - const scope = scopesAfterRelations[relationIndex]; - if (scope !== undefined && token.to < to) { + if (token.to < to) { result.push(Object.freeze({ block: block.id, kind: "join-condition", @@ -792,7 +956,7 @@ export function normalizeNodeSqlParserQueryBindings( if (emitted === null) { return Object.freeze({ reason: "unsupported-shape", status: "unavailable" }); } - const declarations = cteDeclarations(astRoot, tokens); + const declarations = visibleCteDeclarations(tree.root, tokens); if (declarations === null) { return Object.freeze({ reason: "malformed-ast", status: "unavailable" }); } @@ -823,7 +987,7 @@ export function normalizeNodeSqlParserQueryBindings( const publicBlocks = emitted.blocks.map((block) => Object.freeze({ baseScope: 0, - kind: "select" as const, + kind: block.ast.kind, parentBlock: block.parent, range: block.range, }), @@ -848,11 +1012,10 @@ export function normalizeNodeSqlParserQueryBindings( return Object.freeze({ reason: "resource-limit", status: "unavailable" }); } const rawRelation = fromItems[index]; - const site = sites[index]; + const site = phaseValue(sites, index); if ( rawRelation === null || - typeof rawRelation !== "object" || - site === undefined + typeof rawRelation !== "object" ) { return Object.freeze({ reason: "malformed-ast", status: "unavailable" }); } @@ -860,21 +1023,25 @@ export function normalizeNodeSqlParserQueryBindings( const aliasToken = site.aliasToken; const aliasName = aliasToken === null ? null : tokenIdentifier(aliasToken); + const aliasMatches = + aliasText !== null && + aliasName !== null && + identifierMatchesAst(aliasText, aliasName); const alias = - aliasText !== null && aliasName !== null && aliasToken !== null + aliasMatches && aliasName !== null && aliasToken !== null ? Object.freeze({ explicit: site.explicitAlias, - name: Object.freeze({ - quoted: aliasName.quoted, - value: aliasText, - }), + name: aliasName, range: Object.freeze({ from: aliasToken.from, to: aliasToken.to, }), }) : null; - if ((aliasText === null) !== (site.aliasToken === null)) { + if ( + (aliasText === null) !== (site.aliasToken === null) || + (aliasText !== null && aliasName !== null && !aliasMatches) + ) { coverage.relationBindings = "partial"; addIssue(issues, "unsupported-relation-source", { from: site.from, @@ -902,20 +1069,25 @@ export function normalizeNodeSqlParserQueryBindings( ); if (path === null || site.derived) { coverage.relationBindings = "partial"; + addIssue(issues, "unsupported-relation-source", { + from: site.from, + to: site.to, + }); source = Object.freeze({ kind: "unknown", reason: "unsupported-relation-source", }); } else { const last = phaseValue(path, path.length - 1); - const declaration = declarations.get(last.value.toLowerCase()); + const blockDeclarations = declarations.get(block.ast.node); + const declaration = blockDeclarations?.get(identifierKey(last)); source = declaration === undefined ? Object.freeze({ kind: "named", path }) : Object.freeze({ - declarationRange: declaration, + declarationRange: declaration.range, kind: "cte", - name: last, + name: declaration.name, }); } } @@ -942,7 +1114,7 @@ export function normalizeNodeSqlParserQueryBindings( after.push(currentScope); } regions.push(...clauseRegions(tokens, block, currentScope)); - regions.push(...joinRegions(tokens, block, after)); + regions.push(...joinRegions(tokens, block, sites, after)); } regions.sort((left, right) => diff --git a/src/vnext/query-binding-model.ts b/src/vnext/query-binding-model.ts index 924e9da..8b3537b 100644 --- a/src/vnext/query-binding-model.ts +++ b/src/vnext/query-binding-model.ts @@ -358,43 +358,56 @@ function authenticatedItem( return value; } -function isBlockAncestorOrSelf( - model: SqlQueryBindingModel, - ancestor: number, - block: number, -): boolean { - let cursor: number | null = block; - while (cursor !== null) { - if (cursor === ancestor) { - return true; - } - cursor = authenticatedItem(model.blocks, cursor).parentBlock; - } - return false; +interface RelationshipValidationIndex { + readonly ancestors: readonly Uint8Array[]; + readonly scopesIncludingLocal: readonly Uint8Array[]; + readonly scopesWithoutLocal: readonly Uint8Array[]; } -function validateScopeOwners( +function relationshipValidationIndex( model: SqlQueryBindingModel, - scopeIndex: number, - blockIndex: number, - includeLocal: boolean, - subject: string, -): void { - let cursor: number | null = scopeIndex; - while (cursor !== null) { - const scope: SqlRelationScope = - authenticatedItem(model.scopes, cursor); - if (scope.addedBinding !== null) { - const binding = authenticatedItem(model.bindings, scope.addedBinding); - const visible = - isBlockAncestorOrSelf(model, binding.owner, blockIndex) && - (includeLocal || binding.owner !== blockIndex); - if (!visible) { - throw invalid(`${subject} contains an unrelated relation binding`); +): RelationshipValidationIndex { + const ancestors = model.blocks.map((_block, blockIndex) => { + const result = new Uint8Array(model.blocks.length); + let cursor: number | null = blockIndex; + while (cursor !== null) { + result[cursor] = 1; + cursor = authenticatedItem(model.blocks, cursor).parentBlock; + } + return result; + }); + const scopesIncludingLocal: Uint8Array[] = []; + const scopesWithoutLocal: Uint8Array[] = []; + for (let blockIndex = 0; blockIndex < model.blocks.length; blockIndex += 1) { + const includingLocal = new Uint8Array(model.scopes.length); + const withoutLocal = new Uint8Array(model.scopes.length); + const blockAncestors = authenticatedItem(ancestors, blockIndex); + for (let scopeIndex = 0; scopeIndex < model.scopes.length; scopeIndex += 1) { + const scope = authenticatedItem(model.scopes, scopeIndex); + const parentIncluding = scope.parentScope === null || + includingLocal[scope.parentScope] === 1; + const parentWithout = scope.parentScope === null || + withoutLocal[scope.parentScope] === 1; + if (scope.addedBinding === null) { + includingLocal[scopeIndex] = parentIncluding ? 1 : 0; + withoutLocal[scopeIndex] = parentWithout ? 1 : 0; + continue; } + const binding = authenticatedItem(model.bindings, scope.addedBinding); + const ownerVisible = blockAncestors[binding.owner] === 1; + includingLocal[scopeIndex] = + parentIncluding && ownerVisible ? 1 : 0; + withoutLocal[scopeIndex] = + parentWithout && ownerVisible && binding.owner !== blockIndex ? 1 : 0; } - cursor = scope.parentScope; + scopesIncludingLocal.push(includingLocal); + scopesWithoutLocal.push(withoutLocal); } + return { + ancestors, + scopesIncludingLocal, + scopesWithoutLocal, + }; } function wellFormedString(value: string): boolean { @@ -756,6 +769,7 @@ function issueValue( } function validateRelationships(model: SqlQueryBindingModel): void { + const validation = relationshipValidationIndex(model); for (let index = 0; index < model.blocks.length; index += 1) { const block = authenticatedItem(model.blocks, index); if (block.parentBlock !== null) { @@ -764,13 +778,14 @@ function validateRelationships(model: SqlQueryBindingModel): void { throw invalid(`query block ${index} is outside its parent`); } } - validateScopeOwners( - model, - block.baseScope, - index, - false, - `query block ${index} base scope`, - ); + if ( + authenticatedItem(validation.scopesWithoutLocal, index)[block.baseScope] !== + 1 + ) { + throw invalid( + `query block ${index} base scope contains an unrelated relation binding`, + ); + } for (let sibling = 0; sibling < index; sibling += 1) { const other = authenticatedItem(model.blocks, sibling); const overlaps = @@ -778,8 +793,8 @@ function validateRelationships(model: SqlQueryBindingModel): void { other.range.from < block.range.to; if ( overlaps && - !isBlockAncestorOrSelf(model, sibling, index) && - !isBlockAncestorOrSelf(model, index, sibling) + authenticatedItem(validation.ancestors, index)[sibling] !== 1 && + authenticatedItem(validation.ancestors, sibling)[index] !== 1 ) { throw invalid(`query block ${index} overlaps an unrelated block`); } @@ -793,7 +808,10 @@ function validateRelationships(model: SqlQueryBindingModel): void { } if (binding.source.kind === "derived") { const derived = authenticatedItem(model.blocks, binding.source.block); - if (derived.parentBlock !== binding.owner) { + if ( + derived.parentBlock !== binding.owner || + !rangeContains(binding.range, derived.range) + ) { throw invalid(`derived relation binding ${index} has an unrelated block`); } } @@ -827,13 +845,16 @@ function validateRelationships(model: SqlQueryBindingModel): void { if (!rangeContains(block.range, region.range)) { throw invalid(`visibility region ${index} is outside its block`); } - validateScopeOwners( - model, - region.scope, - region.block, - true, - `visibility region ${index} scope`, - ); + if ( + authenticatedItem( + validation.scopesIncludingLocal, + region.block, + )[region.scope] !== 1 + ) { + throw invalid( + `visibility region ${index} scope contains an unrelated relation binding`, + ); + } previousEnd = region.range.to; } } diff --git a/src/vnext/session.ts b/src/vnext/session.ts index bf15af8..628410a 100644 --- a/src/vnext/session.ts +++ b/src/vnext/session.ts @@ -578,6 +578,36 @@ function cancelCompletionTickets( for (const ticket of request.tickets) ticket.cancel(); } +async function raceCatalogResponse( + result: Promise, + timeoutMs: number, +): Promise< + | { readonly kind: "outcome"; readonly outcome: Outcome } + | { readonly kind: "timeout" } +> { + let timer: ReturnType | undefined; + const raced = await Promise.race([ + result.then((outcome) => + Object.freeze({ kind: "outcome" as const, outcome }) + ), + new Promise<{ readonly kind: "timeout" }>((resolve) => { + timer = setTimeout( + () => resolve(Object.freeze({ kind: "timeout" })), + timeoutMs, + ); + }), + ]); + clearTimeout(timer); + return raced; +} + +function remainingCatalogBudget( + startedAt: number, + budgetMs: number, +): number { + return Math.max(0, budgetMs - Math.max(0, performance.now() - startedAt)); +} + function namespaceCompletionList( composition: SqlNamespaceCompletionComposition, ): SqlCompletionList { @@ -1723,15 +1753,13 @@ export class DefaultSqlDocumentSession token: refreshToken, }; this.#activeCompletion = active; + const isCurrent = (): boolean => + this.#activeCompletion === active && + active.cancelReason === null && + snapshot.revision === this.#snapshot.revision; const cancellationIfNotCurrent = (): SqlCompletionResult | null => { - if ( - this.#activeCompletion === active && - active.cancelReason === null && - snapshot.revision === this.#snapshot.revision - ) { - return null; - } + if (isCurrent()) return null; cancelPrevious(); return completionCancellation( snapshot.revision, @@ -1882,11 +1910,7 @@ export class DefaultSqlDocumentSession status: "unavailable", }); } - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + if (!isCurrent()) { return completionCancellation( snapshot.revision, completionCancellationReason(active), @@ -1898,29 +1922,14 @@ export class DefaultSqlDocumentSession searchPaths: catalog.searchPaths, }); active.tickets.add(ticket); - let responseTimer: - | ReturnType - | undefined; - const raced = await Promise.race([ - ticket.result.then((outcome) => - Object.freeze({ - kind: "outcome" as const, - outcome, - }), + const raced = await raceCatalogResponse( + ticket.result, + remainingCatalogBudget( + completionStartedAt, + this.#catalogResponseBudgetMs, ), - new Promise<{ readonly kind: "timeout" }>((resolve) => { - responseTimer = setTimeout( - () => resolve(Object.freeze({ kind: "timeout" })), - this.#catalogResponseBudgetMs, - ); - }), - ]); - clearTimeout(responseTimer); - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + ); + if (!isCurrent()) { ticket.cancel(); return completionCancellation( snapshot.revision, @@ -2005,11 +2014,7 @@ export class DefaultSqlDocumentSession status: "unavailable", }); } - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + if (!isCurrent()) { return completionCancellation( snapshot.revision, completionCancellationReason(active), @@ -2036,11 +2041,7 @@ export class DefaultSqlDocumentSession status: "unavailable", }); } else { - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + if (!isCurrent()) { cancelPrevious(); return completionCancellation( snapshot.revision, @@ -2058,37 +2059,14 @@ export class DefaultSqlDocumentSession this.#refreshIntent = null; } active.tickets.add(ticket); - const elapsed = Math.max( - 0, - performance.now() - completionStartedAt, - ); - const remaining = Math.max( - 0, - this.#catalogResponseBudgetMs - elapsed, - ); - let responseTimer: - | ReturnType - | undefined; - const raced = await Promise.race([ - ticket.result.then((outcome) => - Object.freeze({ - kind: "outcome" as const, - outcome, - }), + const raced = await raceCatalogResponse( + ticket.result, + remainingCatalogBudget( + completionStartedAt, + this.#catalogResponseBudgetMs, ), - new Promise<{ readonly kind: "timeout" }>((resolve) => { - responseTimer = setTimeout( - () => resolve(Object.freeze({ kind: "timeout" })), - remaining, - ); - }), - ]); - clearTimeout(responseTimer); - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + ); + if (!isCurrent()) { ticket.cancel(); return completionCancellation( snapshot.revision, @@ -2251,37 +2229,14 @@ export class DefaultSqlDocumentSession ), ); active.tickets.add(ticket); - const elapsed = Math.max( - 0, - performance.now() - completionStartedAt, - ); - const remaining = Math.max( - 0, - this.#catalogResponseBudgetMs - elapsed, - ); - let responseTimer: - | ReturnType - | undefined; - const raced = await Promise.race([ - ticket.result.then((outcome) => - Object.freeze({ - kind: "outcome" as const, - outcome, - }), + const raced = await raceCatalogResponse( + ticket.result, + remainingCatalogBudget( + completionStartedAt, + this.#catalogResponseBudgetMs, ), - new Promise<{ readonly kind: "timeout" }>((resolve) => { - responseTimer = setTimeout( - () => resolve(Object.freeze({ kind: "timeout" })), - remaining, - ); - }), - ]); - clearTimeout(responseTimer); - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + ); + if (!isCurrent()) { ticket.cancel(); return completionCancellation( snapshot.revision, @@ -2356,11 +2311,7 @@ export class DefaultSqlDocumentSession replacementRange, statementOffset, }); - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + if (!isCurrent()) { return completionCancellation( snapshot.revision, completionCancellationReason(active), diff --git a/src/vnext/types.ts b/src/vnext/types.ts index 0a081ba..a0582b4 100644 --- a/src/vnext/types.ts +++ b/src/vnext/types.ts @@ -1,5 +1,15 @@ const revisionBrand: unique symbol = Symbol("SqlRevision"); +export function isDataArray( + value: unknown, +): value is readonly unknown[] { + try { + return Array.isArray(value); + } catch { + return false; + } +} + /** Immutable identity issued by a document session. */ export interface SqlRevision { readonly [revisionBrand]: "SqlRevision"; From f75f02279e6ef9c23fbe1d539ace26fd4e2d6c43 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 13:39:46 +0800 Subject: [PATCH 23/25] fix(vnext): close recursive and lateral scope gaps --- src/vnext/__tests__/column-query-site.test.ts | 14 ++ .../node-sql-parser-query-bindings.test.ts | 181 ++++++++++++++++++ src/vnext/__tests__/session.test.ts | 56 ++++++ src/vnext/column-query-site.ts | 60 ++++-- src/vnext/node-sql-parser-query-bindings.ts | 137 ++++++++++--- 5 files changed, 405 insertions(+), 43 deletions(-) diff --git a/src/vnext/__tests__/column-query-site.test.ts b/src/vnext/__tests__/column-query-site.test.ts index 1bed1a9..91d9d94 100644 --- a/src/vnext/__tests__/column-query-site.test.ts +++ b/src/vnext/__tests__/column-query-site.test.ts @@ -123,6 +123,20 @@ describe("recognizeSqlColumnQuerySite", () => { )).toEqual(["inner_table"]); }); + it("correlates PostgreSQL LATERAL tables only to preceding relations", () => { + for (const expression of ["u.na|", "na|"]) { + const result = ready( + analyze( + `SELECT * FROM users u, LATERAL (SELECT ${expression} FROM orders o) x, secrets s`, + ), + ); + + expect(result.relations.map((relation) => + relation.alias?.value + )).toEqual(["o", "u"]); + } + }); + it("limits correlation in a JOIN condition to prior relations", () => { const result = ready( analyze( diff --git a/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts b/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts index 672f973..dfdee5a 100644 --- a/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts +++ b/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts @@ -275,6 +275,187 @@ describe("node-sql-parser query binding normalization", () => { }); }); + it("authenticates recursive CTE self-visibility", () => { + const body = { + from: [relation("self")], + type: "select", + }; + const result = normalize( + { + from: [relation("self")], + type: "select", + with: [{ + name: { value: "self" }, + recursive: true, + stmt: body, + }], + }, + "WITH RECURSIVE self AS (SELECT * FROM self) SELECT * FROM self", + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte", name: { value: "self" } } }, + { owner: 1, source: { kind: "cte", name: { value: "self" } } }, + ], + coverage: { relationBindings: "complete" }, + }, + status: "ready", + }); + if (result.status === "ready") { + expect(result.model.issues).not.toContainEqual( + expect.objectContaining({ code: "recursive-cte-uncertainty" }), + ); + } + }); + + it("fails closed when recursive lexical and AST evidence disagree", () => { + const body = { + from: [relation("self")], + type: "select", + }; + const lexicalOnly = normalize( + { + from: [relation("self")], + type: "select", + with: [{ name: { value: "self" }, stmt: body }], + }, + "WITH RECURSIVE self AS (SELECT * FROM self) SELECT * FROM self", + ); + const astOnly = normalize( + { + from: [relation("self")], + type: "select", + with: [{ + name: { value: "self" }, + recursive: true, + stmt: body, + }], + }, + "WITH self AS (SELECT * FROM self) SELECT * FROM self", + ); + for (const result of [lexicalOnly, astOnly]) { + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte" } }, + { + owner: 1, + source: { kind: "unknown", reason: "unknown-correlation" }, + }, + ], + coverage: { relationBindings: "partial" }, + issues: [{ code: "recursive-cte-uncertainty" }], + }, + status: "ready", + }); + } + }); + + it("keeps non-recursive self and forward names outside CTE visibility", () => { + const self = normalize( + { + from: [relation("self")], + type: "select", + with: [{ + name: { value: "self" }, + stmt: { from: [relation("self")], type: "select" }, + }], + }, + "WITH self AS (SELECT * FROM self) SELECT * FROM self", + ); + expect(self).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte" } }, + { owner: 1, source: { kind: "named", path: [{ value: "self" }] } }, + ], + coverage: { relationBindings: "complete" }, + }, + status: "ready", + }); + }); + + it("marks recursive forward and mutual references uncertain", () => { + const first = { + from: [relation("later")], + type: "select", + }; + const forward = normalize( + { + from: [relation("first")], + type: "select", + with: [ + { + name: { value: "first" }, + recursive: true, + stmt: first, + }, + { + name: { value: "later" }, + stmt: { from: [relation("events")], type: "select" }, + }, + ], + }, + "WITH RECURSIVE first AS (SELECT * FROM later), " + + "later AS (SELECT * FROM events) SELECT * FROM first", + ); + expect(forward).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte", name: { value: "first" } } }, + { + owner: 1, + source: { kind: "unknown", reason: "unknown-correlation" }, + }, + { + owner: 2, + source: { kind: "named", path: [{ value: "events" }] }, + }, + ], + coverage: { relationBindings: "partial" }, + issues: [{ code: "recursive-cte-uncertainty" }], + }, + status: "ready", + }); + + const later = { + from: [relation("first")], + type: "select", + }; + const result = normalize( + { + from: [relation("first")], + type: "select", + with: [ + { + name: { value: "first" }, + recursive: true, + stmt: first, + }, + { name: { value: "later" }, stmt: later }, + ], + }, + "WITH RECURSIVE first AS (SELECT * FROM later), " + + "later AS (SELECT * FROM first) SELECT * FROM first", + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte", name: { value: "first" } } }, + { + owner: 1, + source: { kind: "unknown", reason: "unknown-correlation" }, + }, + { owner: 2, source: { kind: "cte", name: { value: "first" } } }, + ], + coverage: { relationBindings: "partial" }, + issues: [{ code: "recursive-cte-uncertainty" }], + }, + status: "ready", + }); + }); + it("resolves a nested WITH inside its derived query block", () => { const nestedCte = { from: [relation("events")], diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index 145d2e8..4666064 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -1447,6 +1447,62 @@ describe("column completion session integration", () => { service.dispose(); }); + it("completes PostgreSQL LATERAL references without loading later siblings", async () => { + for (const expression of ["u.", "na"]) { + const requests: Parameters< + SqlColumnCatalogProvider["loadColumns"] + >[0][] = []; + const service = createSqlLanguageService({ + catalog: relationCatalog, + columns: { + id: "columns", + loadColumns: async (request) => { + requests.push(request); + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: request.relations.map((relation, index) => ({ + columns: [{ + columnEntityId: `column-${index}`, + identifier: { quoted: false, value: "name" }, + insertText: "name", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: `relation-${index}`, + requestKey: relation.requestKey, + status: "ready", + })), + }; + }, + }, + dialects: [postgres], + }); + const text = + `SELECT * FROM users u, LATERAL (SELECT ${expression} FROM orders o) x, secrets s`; + const session = service.openDocument({ + context: { + catalog: { scope: `connection:lateral:${expression}` }, + dialect: "postgresql", + engine: "local", + }, + text, + }); + const position = text.indexOf(expression) + expression.length; + + await expect(session.complete({ + position, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ status: "ready" }); + expect(requests).toHaveLength(1); + expect(requests[0]?.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(expression === "u." + ? ["users"] + : ["orders", "users"]); + service.dispose(); + } + }); + it("never resolves a visible CTE as a physical relation", async () => { let calls = 0; const service = serviceWithColumns(async () => { diff --git a/src/vnext/column-query-site.ts b/src/vnext/column-query-site.ts index c66a0a5..e727dcb 100644 --- a/src/vnext/column-query-site.ts +++ b/src/vnext/column-query-site.ts @@ -408,6 +408,7 @@ function collectRelations( tokens: readonly Token[], selectIndex: number, visibilityPosition: number, + precedingOnly: boolean, relations: SqlColumnQueryRelation[], issues: Set, ): boolean { @@ -429,7 +430,7 @@ function collectRelations( ) { const token = tokens[index]!; if ( - clause === "join-condition" && + (precedingOnly || clause === "join-condition") && token.from >= visibilityPosition ) { break; @@ -471,11 +472,18 @@ function collectRelations( punctuation(source, token, ",")); if (!startsRelation) continue; inFrom = true; + let relationIndex = index + 1; + if ( + dialect.id === "postgresql" && + word(source, tokens[relationIndex]) === "lateral" + ) { + relationIndex += 1; + } const parsed = parseNamedRelation( source, dialect, tokens, - index + 1, + relationIndex, selectDepth, ); if (parsed.issue !== null) issues.add(parsed.issue); @@ -501,7 +509,7 @@ function openingBefore( token.depth === depth && punctuation(source, token, "(") ) { - return token.from; + return index; } } return -1; @@ -509,39 +517,47 @@ function openingBefore( function correlatedParentSelectIndex( source: SqlSourceSnapshot, + dialect: SqlRelationDialectRuntime, tokens: readonly Token[], childIndex: number, -): number { +): readonly [selectIndex: number, precedingOnly: boolean] | null { const child = tokens[childIndex]; - if (!child || child.depth === 0) return -1; + if (!child || child.depth === 0) return null; for (let depth = child.depth - 1; depth >= 0; depth -= 1) { const lowerBound = depth === 0 ? -1 - : openingBefore(source, tokens, childIndex, depth - 1); + : tokens[ + openingBefore(source, tokens, childIndex, depth - 1) + ]?.from ?? -1; for (let index = childIndex - 1; index >= 0; index -= 1) { const token = tokens[index]!; if (token.from <= lowerBound) break; if (token.depth === depth && word(source, token) === "select") { - const opening = openingBefore( + const openingIndex = openingBefore( source, tokens, childIndex, depth, ); - return opening > token.from && - clauseAt( - source, - tokens, - index, - depth, - opening, - ) !== "from" - ? index - : -1; + const opening = tokens[openingIndex]?.from ?? -1; + if (opening <= token.from) return null; + const inFrom = + clauseAt( + source, + tokens, + index, + depth, + opening, + ) === "from"; + if (!inFrom) return [index, false]; + return dialect.id === "postgresql" && + word(source, tokens[openingIndex - 1]) === "lateral" + ? [index, true] + : null; } } } - return -1; + return null; } export function recognizeSqlColumnQuerySite( @@ -617,6 +633,7 @@ export function recognizeSqlColumnQuerySite( tokens, selectIndex, position, + false, relations, issues, ) @@ -625,12 +642,14 @@ export function recognizeSqlColumnQuerySite( } let childIndex = selectIndex; for (;;) { - const parentIndex = correlatedParentSelectIndex( + const parent = correlatedParentSelectIndex( source, + dialect, tokens, childIndex, ); - if (parentIndex < 0) break; + if (parent === null) break; + const [parentIndex, precedingOnly] = parent; const childPosition = tokens[childIndex]?.from; if ( childPosition === undefined || @@ -640,6 +659,7 @@ export function recognizeSqlColumnQuerySite( tokens, parentIndex, childPosition, + precedingOnly, relations, issues, ) diff --git a/src/vnext/node-sql-parser-query-bindings.ts b/src/vnext/node-sql-parser-query-bindings.ts index 83a9776..6053f6d 100644 --- a/src/vnext/node-sql-parser-query-bindings.ts +++ b/src/vnext/node-sql-parser-query-bindings.ts @@ -643,6 +643,16 @@ interface CteDeclaration { readonly range: SqlTextRange; } +interface CteClause { + readonly declarations: readonly CteDeclaration[]; + readonly recursion: "authenticated" | "none" | "uncertain"; +} + +interface CteEnvironment { + readonly uncertain: ReadonlySet; + readonly visible: ReadonlyMap; +} + interface IdentifierToken { readonly identifier: SqlIdentifierComponent; readonly token: Token; @@ -685,10 +695,21 @@ function findIdentifierToken( return null; } +function astRecursiveMarker(item: object): boolean | null { + const property = ownProperty(item, "recursive"); + if (property.kind === "missing") { + return false; + } + return property.kind === "value" && + typeof property.value === "boolean" + ? property.value + : null; +} + function ownCteDeclarations( ast: AstSelect, tokens: readonly Token[], -): readonly CteDeclaration[] | null { +): CteClause | null { const withItems = arrayProperty(ast.node, "with"); if (withItems === null) { return null; @@ -697,6 +718,8 @@ function ownCteDeclarations( return null; } const declarations: CteDeclaration[] = []; + let astRecursive = false; + let validMarkers = true; let searchFrom = assignedSkeleton(ast).from; for (let index = 0; index < withItems.length; index += 1) { const item = withItems[index]; @@ -704,6 +727,12 @@ function ownCteDeclarations( if (item === null || typeof item !== "object") { return null; } + const recursive = astRecursiveMarker(item); + if (recursive === null || (index > 0 && recursive)) { + validMarkers = false; + } else if (index === 0) { + astRecursive = recursive; + } const nameObject = objectProperty(item, "name"); const name = nameObject === null ? stringProperty(item, "name") @@ -730,40 +759,90 @@ function ownCteDeclarations( })); searchFrom = assignedSkeleton(child).to; } - return Object.freeze(declarations); + if (declarations.length === 0) { + return Object.freeze({ + declarations: Object.freeze(declarations), + recursion: "none", + }); + } + const first = phaseValue(declarations, 0); + const withIndex = tokens.findIndex((token) => + token.from >= assignedSkeleton(ast).from && + token.from < first.range.from && + token.depth === assignedSkeleton(ast).depth && + word(token, "with") + ); + if (withIndex < 0) { + return null; + } + const lexicalRecursive = word(tokens[withIndex + 1], "recursive"); + return Object.freeze({ + declarations: Object.freeze(declarations), + recursion: + validMarkers && lexicalRecursive === astRecursive + ? lexicalRecursive ? "authenticated" : "none" + : "uncertain", + }); } function visibleCteDeclarations( root: AstSelect, tokens: readonly Token[], -): ReadonlyMap> | null { - const result = new Map>(); +): ReadonlyMap | null { + const result = new Map(); function visit( ast: AstSelect, - inherited: ReadonlyMap, + inherited: CteEnvironment, ): boolean { - const own = ownCteDeclarations(ast, tokens); - if (own === null) { + const clause = ownCteDeclarations(ast, tokens); + if (clause === null) { return false; } - const visible = new Map(inherited); + const visible = new Map(inherited.visible); + const uncertain = new Set(inherited.uncertain); for (let index = 0; index < ast.cteChildren.length; index += 1) { const child = phaseValue(ast.cteChildren, index); - if (!visit(child, visible)) { + const declaration = phaseValue(clause.declarations, index); + const childVisible = new Map(visible); + const childUncertain = new Set(uncertain); + if (clause.recursion === "authenticated") { + childVisible.set(identifierKey(declaration.name), declaration); + for ( + let forward = index + 1; + forward < clause.declarations.length; + forward += 1 + ) { + childUncertain.add( + identifierKey(phaseValue(clause.declarations, forward).name), + ); + } + } else if (clause.recursion === "uncertain") { + for (const candidate of clause.declarations) { + childUncertain.add(identifierKey(candidate.name)); + } + } + if (!visit(child, { + uncertain: childUncertain, + visible: childVisible, + })) { return false; } - const declaration = phaseValue(own, index); visible.set(identifierKey(declaration.name), declaration); + uncertain.delete(identifierKey(declaration.name)); } - result.set(ast.node, visible); + const environment = { uncertain, visible }; + result.set(ast.node, environment); for (const child of ast.derivedChildren) { - if (!visit(child, visible)) { + if (!visit(child, environment)) { return false; } } return true; } - return visit(root, new Map()) ? result : null; + return visit(root, { + uncertain: new Set(), + visible: new Map(), + }) ? result : null; } function addIssue( @@ -1079,16 +1158,28 @@ export function normalizeNodeSqlParserQueryBindings( }); } else { const last = phaseValue(path, path.length - 1); - const blockDeclarations = declarations.get(block.ast.node); - const declaration = blockDeclarations?.get(identifierKey(last)); - source = - declaration === undefined - ? Object.freeze({ kind: "named", path }) - : Object.freeze({ - declarationRange: declaration.range, - kind: "cte", - name: declaration.name, - }); + const environment = declarations.get(block.ast.node); + const key = identifierKey(last); + const declaration = environment?.visible.get(key); + if (declaration !== undefined) { + source = Object.freeze({ + declarationRange: declaration.range, + kind: "cte", + name: declaration.name, + }); + } else if (environment?.uncertain.has(key)) { + coverage.relationBindings = "partial"; + addIssue(issues, "recursive-cte-uncertainty", { + from: site.from, + to: site.to, + }); + source = Object.freeze({ + kind: "unknown", + reason: "unknown-correlation", + }); + } else { + source = Object.freeze({ kind: "named", path }); + } } } const bindingIndex = bindings.length; From e74bc5867c0ac837b38a92bd3c6baee381955943 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 13:48:52 +0800 Subject: [PATCH 24/25] test(vnext): isolate catalog deadline timing --- src/vnext/__tests__/relation-catalog-search-work.test.ts | 4 +++- src/vnext/__tests__/session.test.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/vnext/__tests__/relation-catalog-search-work.test.ts b/src/vnext/__tests__/relation-catalog-search-work.test.ts index a6914c7..f6dbbff 100644 --- a/src/vnext/__tests__/relation-catalog-search-work.test.ts +++ b/src/vnext/__tests__/relation-catalog-search-work.test.ts @@ -1045,7 +1045,9 @@ describe("catalog search ownership and active-slot lifecycle", () => { it("starts observer-only queued work when an active slot becomes free", async () => { const provider = providerHarness(); - const service = coordinator(provider.captured); + const service = coordinator(provider.captured, { + deadlineScheduler: new ManualDeadlineScheduler(), + }); const active = Array.from( { length: MAX_CATALOG_ACTIVE_SEARCH_WORK }, (_, index) => diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index 4666064..20e351b 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -600,7 +600,7 @@ describe("relation completion session integration", () => { }); it("does not correlate catalog invalidation after a soft lease expires", async () => { - vi.useFakeTimers(); + vi.useFakeTimers({ shouldAdvanceTime: true }); let service: | ReturnType> | undefined; From 8eaae1edc78855237bbb4ce99f3e98edbfdfd762 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 26 Jul 2026 13:56:24 +0800 Subject: [PATCH 25/25] fix(vnext): close nested scope review gaps --- src/vnext/__tests__/column-query-site.test.ts | 31 +++-- .../node-sql-parser-query-bindings.test.ts | 43 +++++++ src/vnext/__tests__/session.test.ts | 106 +++++++++--------- src/vnext/column-query-site.ts | 4 +- src/vnext/node-sql-parser-query-bindings.ts | 10 +- 5 files changed, 127 insertions(+), 67 deletions(-) diff --git a/src/vnext/__tests__/column-query-site.test.ts b/src/vnext/__tests__/column-query-site.test.ts index 91d9d94..abddfc7 100644 --- a/src/vnext/__tests__/column-query-site.test.ts +++ b/src/vnext/__tests__/column-query-site.test.ts @@ -7,6 +7,7 @@ import { } from "../column-query-site.js"; import { BIGQUERY_SQL_RELATION_DIALECT, + DUCKDB_SQL_RELATION_DIALECT, POSTGRESQL_SQL_RELATION_DIALECT, type SqlRelationDialectRuntime, } from "../relation-dialect.js"; @@ -123,17 +124,25 @@ describe("recognizeSqlColumnQuerySite", () => { )).toEqual(["inner_table"]); }); - it("correlates PostgreSQL LATERAL tables only to preceding relations", () => { - for (const expression of ["u.na|", "na|"]) { - const result = ready( - analyze( - `SELECT * FROM users u, LATERAL (SELECT ${expression} FROM orders o) x, secrets s`, - ), - ); - - expect(result.relations.map((relation) => - relation.alias?.value - )).toEqual(["o", "u"]); + it("correlates LATERAL tables only to preceding relations", () => { + for ( + const dialect of [ + POSTGRESQL_SQL_RELATION_DIALECT, + DUCKDB_SQL_RELATION_DIALECT, + ] + ) { + for (const expression of ["u.na|", "na|"]) { + const result = ready( + analyze( + `SELECT * FROM users u, LATERAL (SELECT ${expression} FROM orders o) x, secrets s`, + { dialect }, + ), + ); + + expect(result.relations.map((relation) => + relation.alias?.value + )).toEqual(["o", "u"]); + } } }); diff --git a/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts b/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts index dfdee5a..24b2c7f 100644 --- a/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts +++ b/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts @@ -352,6 +352,49 @@ describe("node-sql-parser query binding normalization", () => { } }); + it("does not resolve an uncertain local CTE through an outer namesake", () => { + const nested = { + from: [relation("shadow")], + type: "select", + with: [{ + name: { value: "shadow" }, + stmt: { from: [relation("shadow")], type: "select" }, + }], + }; + const result = normalize( + { + from: [{ as: "d", expr: { ast: nested, parentheses: true } }], + type: "select", + with: [{ + name: { value: "shadow" }, + stmt: { from: [relation("events")], type: "select" }, + }], + }, + "WITH shadow AS (SELECT * FROM events) SELECT * FROM (" + + "WITH RECURSIVE shadow AS (SELECT * FROM shadow) " + + "SELECT * FROM shadow) d", + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { block: 2, kind: "derived" } }, + { + owner: 1, + source: { kind: "named", path: [{ value: "events" }] }, + }, + { owner: 2, source: { kind: "cte", name: { value: "shadow" } } }, + { + owner: 3, + source: { kind: "unknown", reason: "unknown-correlation" }, + }, + ], + coverage: { relationBindings: "partial" }, + issues: [{ code: "recursive-cte-uncertainty" }], + }, + status: "ready", + }); + }); + it("keeps non-recursive self and forward names outside CTE visibility", () => { const self = normalize( { diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index 20e351b..c8d3ead 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -1447,59 +1447,63 @@ describe("column completion session integration", () => { service.dispose(); }); - it("completes PostgreSQL LATERAL references without loading later siblings", async () => { - for (const expression of ["u.", "na"]) { - const requests: Parameters< - SqlColumnCatalogProvider["loadColumns"] - >[0][] = []; - const service = createSqlLanguageService({ - catalog: relationCatalog, - columns: { - id: "columns", - loadColumns: async (request) => { - requests.push(request); - return { - epoch: { generation: 1, token: "epoch-1" }, - relations: request.relations.map((relation, index) => ({ - columns: [{ - columnEntityId: `column-${index}`, - identifier: { quoted: false, value: "name" }, - insertText: "name", - ordinal: 0, - }], - coverage: "complete", - relationEntityId: `relation-${index}`, - requestKey: relation.requestKey, - status: "ready", - })), - }; + it("completes LATERAL references without loading later siblings", async () => { + for (const dialect of [postgres, duckdb]) { + for (const expression of ["u.", "na"]) { + const requests: Parameters< + SqlColumnCatalogProvider["loadColumns"] + >[0][] = []; + const service = createSqlLanguageService({ + catalog: relationCatalog, + columns: { + id: "columns", + loadColumns: async (request) => { + requests.push(request); + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: request.relations.map((relation, index) => ({ + columns: [{ + columnEntityId: `column-${index}`, + identifier: { quoted: false, value: "name" }, + insertText: "name", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: `relation-${index}`, + requestKey: relation.requestKey, + status: "ready", + })), + }; + }, }, - }, - dialects: [postgres], - }); - const text = - `SELECT * FROM users u, LATERAL (SELECT ${expression} FROM orders o) x, secrets s`; - const session = service.openDocument({ - context: { - catalog: { scope: `connection:lateral:${expression}` }, - dialect: "postgresql", - engine: "local", - }, - text, - }); - const position = text.indexOf(expression) + expression.length; + dialects: [dialect], + }); + const text = + `SELECT * FROM users u, LATERAL (SELECT ${expression} FROM orders o) x, secrets s`; + const session = service.openDocument({ + context: { + catalog: { + scope: `connection:lateral:${dialect.id}:${expression}`, + }, + dialect: dialect.id, + engine: "local", + }, + text, + }); + const position = text.indexOf(expression) + expression.length; - await expect(session.complete({ - position, - trigger: { kind: "invoked" }, - })).resolves.toMatchObject({ status: "ready" }); - expect(requests).toHaveLength(1); - expect(requests[0]?.relations.map((relation) => - relation.path.at(-1)?.value - )).toEqual(expression === "u." - ? ["users"] - : ["orders", "users"]); - service.dispose(); + await expect(session.complete({ + position, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ status: "ready" }); + expect(requests).toHaveLength(1); + expect(requests[0]?.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(expression === "u." + ? ["users"] + : ["orders", "users"]); + service.dispose(); + } } }); diff --git a/src/vnext/column-query-site.ts b/src/vnext/column-query-site.ts index e727dcb..dc8e8a6 100644 --- a/src/vnext/column-query-site.ts +++ b/src/vnext/column-query-site.ts @@ -474,7 +474,7 @@ function collectRelations( inFrom = true; let relationIndex = index + 1; if ( - dialect.id === "postgresql" && + (dialect.id === "postgresql" || dialect.id === "duckdb") && word(source, tokens[relationIndex]) === "lateral" ) { relationIndex += 1; @@ -550,7 +550,7 @@ function correlatedParentSelectIndex( opening, ) === "from"; if (!inFrom) return [index, false]; - return dialect.id === "postgresql" && + return (dialect.id === "postgresql" || dialect.id === "duckdb") && word(source, tokens[openingIndex - 1]) === "lateral" ? [index, true] : null; diff --git a/src/vnext/node-sql-parser-query-bindings.ts b/src/vnext/node-sql-parser-query-bindings.ts index 6053f6d..786849e 100644 --- a/src/vnext/node-sql-parser-query-bindings.ts +++ b/src/vnext/node-sql-parser-query-bindings.ts @@ -812,13 +812,17 @@ function visibleCteDeclarations( forward < clause.declarations.length; forward += 1 ) { - childUncertain.add( - identifierKey(phaseValue(clause.declarations, forward).name), + const key = identifierKey( + phaseValue(clause.declarations, forward).name, ); + childUncertain.add(key); + childVisible.delete(key); } } else if (clause.recursion === "uncertain") { for (const candidate of clause.declarations) { - childUncertain.add(identifierKey(candidate.name)); + const key = identifierKey(candidate.name); + childUncertain.add(key); + childVisible.delete(key); } } if (!visit(child, {