diff --git a/docs/vnext/capability-charter.md b/docs/vnext/capability-charter.md index d6353aa..d04c8dd 100644 --- a/docs/vnext/capability-charter.md +++ b/docs/vnext/capability-charter.md @@ -239,6 +239,7 @@ count and estimated retained bytes. Provisional compressed bundle budgets: +- Framework-independent core with relation completion: 48 KiB - Core plus CodeMirror adapter, excluding parser and dialect data: 75 KiB - Each ordinary dialect module: 25 KiB - Optional `node-sql-parser` chunk: no regression from its recorded baseline diff --git a/docs/vnext/session-primitives.md b/docs/vnext/session-primitives.md index 6c200d3..6018722 100644 --- a/docs/vnext/session-primitives.md +++ b/docs/vnext/session-primitives.md @@ -3,10 +3,10 @@ Status: experimental walking skeleton Import: `@marimo-team/codemirror-sql/vnext` -This entry point currently provides document ownership, atomic text/context -updates, opaque revisions, dialect registration, and lifecycle management. It -does not yet provide parsing, completion, diagnostics, hover, or navigation. -Those methods will be added only with working vertical slices. +This entry point provides document ownership, atomic text/context updates, +opaque revisions, dialect registration, lifecycle management, and +parser-independent relation completion. Diagnostics, hover, navigation, and +general expression completion are not yet available. See [source coordinates](./source-coordinates.md) for the shared UTF-16 range contract and the internal immutable source-snapshot model. @@ -56,6 +56,73 @@ Use `{ kind: "replace", text }` for full replacement and also supplies the complete embedded-region set for its resulting text. A region-only transaction can replace or clear that set without a fake text edit. +## Relation completion + +Completion works without a catalog for visible CTEs. A service can also own one +shared asynchronous relation-catalog provider: + +```ts +const service = createSqlLanguageService({ + catalog: { + id: "app-catalog", + search: async (request, signal) => { + signal.throwIfAborted(); + return { + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }; + }, + }, + completion: { catalogResponseBudgetMs: 40 }, + dialects: [duckdbDialect()], +}); + +const session = service.openDocument({ + context: { + dialect: "duckdb", + engine: "local", + catalog: { scope: "connection-incarnation:1" }, + }, + text: "SELECT * FROM ", +}); + +const subscription = session.onDidChange(({ reason }) => { + if (reason === "catalog" || reason === "catalog-availability") { + // Ask the editor adapter to request completion again. + } +}); + +const result = await session.complete({ + position: 14, + trigger: { kind: "invoked" }, +}); + +if (result.status === "ready" && session.isCurrent(result.revision)) { + for (const item of result.value.items) { + // Apply item.edit in the original document's UTF-16 coordinates. + } +} + +subscription.dispose(); +session.dispose(); +service.dispose(); +``` + +The catalog `scope` identifies a live connection incarnation, not a reusable +display name. Search responses distinguish complete, partial, paginated, +loading, and failed evidence. A ready result can therefore be incomplete; its +closed `issues` explain why, while `sources` reports catalog outcomes without +exposing provider errors or internal epochs. + +The interactive catalog wait is bounded from the start of `complete()`. When +the budget expires, the session returns local evidence with a +`catalog-loading` issue and a checked remaining intent lease. Compatible +readiness advances the session revision and emits `catalog-availability`; +higher provider epochs emit `catalog`. Consumers must request completion again +and apply only results whose revision remains current. + ## Dialect registration `duckdbDialect()`, `postgresDialect()`, `bigQueryDialect()`, and diff --git a/package.json b/package.json index 401581f..989463c 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "bench:local-relation-site": "vitest bench --run src/vnext/__tests__/local-relation-site.bench.ts", "bench:parser-adapter": "vitest bench --run src/vnext/__tests__/node-sql-parser-adapter.bench.ts", "bench:query-site": "vitest bench --run src/vnext/__tests__/query-site.bench.ts", + "bench:session-completion": "vitest bench --run src/vnext/__tests__/session-completion.bench.ts", "bench:statement-index": "vitest bench --run src/vnext/__tests__/statement-index.bench.ts", "test:package": "node ./scripts/clean.mjs && tsc && node ./scripts/package-smoke.mjs", "demo": "vite build", diff --git a/scripts/worker-placement.mjs b/scripts/worker-placement.mjs index 351e535..fb6e692 100644 --- a/scripts/worker-placement.mjs +++ b/scripts/worker-placement.mjs @@ -35,11 +35,11 @@ const PARSER_MARKERS = [ "tableList", ]; const BIGQUERY_GZIP_LIMIT = 50 * 1024; -const CORE_TOTAL_GZIP_LIMIT = 16 * 1024; -const CORE_TOTAL_RAW_LIMIT = 52 * 1024; +const CORE_TOTAL_GZIP_LIMIT = 48 * 1024; +const CORE_TOTAL_RAW_LIMIT = 180 * 1024; const POSTGRESQL_GZIP_LIMIT = 68 * 1024; -const WORKER_TOTAL_GZIP_LIMIT = 128 * 1024; -const WORKER_TOTAL_RAW_LIMIT = 570 * 1024; +const WORKER_TOTAL_GZIP_LIMIT = 160 * 1024; +const WORKER_TOTAL_RAW_LIMIT = 700 * 1024; const MIME_TYPES = new Map([ [".css", "text/css; charset=utf-8"], [".html", "text/html; charset=utf-8"], diff --git a/src/vnext/__tests__/relation-completion.test.ts b/src/vnext/__tests__/relation-completion.test.ts new file mode 100644 index 0000000..8a25702 --- /dev/null +++ b/src/vnext/__tests__/relation-completion.test.ts @@ -0,0 +1,652 @@ +import { describe, expect, it } from "vitest"; +import type { + SqlValidatedCatalogRelation, + SqlValidatedCatalogSearchResponse, +} from "../relation-catalog-boundary.js"; +import { + composeSqlRelationCompletion, + MAX_RELATION_COMPLETION_RESULTS, + type SqlComposableCatalogOutcome, +} from "../relation-completion.js"; +import { + analyzeSqlLocalRelationSite, + prepareSqlLocalRelationStatement, + type SqlLocalRelationSiteResult, +} from "../local-relation-site.js"; +import { + POSTGRESQL_SQL_RELATION_DIALECT, + type SqlRelationDialectRuntime, +} from "../relation-dialect.js"; +import { + createIdentitySqlSource, +} from "../source.js"; +import { + buildSqlStatementIndex, + findSqlStatementSlot, +} from "../statement-index.js"; +import type { + SqlCanonicalRelationPath, + SqlCatalogRelationKind, + SqlCatalogReadyCoverage, + SqlCompletionList, + SqlRelationCompletionDialectRuntime, +} from "../relation-completion-types.js"; +import type { + SqlTextRange, +} from "../types.js"; + +interface MarkedLocalSite { + readonly localSite: Extract< + SqlLocalRelationSiteResult, + { readonly status: "ready" } + >; + readonly replacementRange: SqlTextRange; + readonly statementOffset: number; +} + +function markedLocalSite(marked: string): MarkedLocalSite { + const position = marked.indexOf("|"); + if ( + position < 0 || + marked.indexOf("|", position + 1) >= 0 + ) { + throw new Error("Expected exactly one cursor marker"); + } + const text = + marked.slice(0, position) + marked.slice(position + 1); + const source = createIdentitySqlSource(text); + const index = buildSqlStatementIndex( + source.analysisText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot(index, position, "left"); + const prepared = prepareSqlLocalRelationStatement( + source, + index, + slot, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + if (prepared.status !== "ready") { + throw new Error(`Local statement unavailable: ${prepared.reason}`); + } + const localSite = analyzeSqlLocalRelationSite( + prepared.statement, + position, + ); + if (localSite.status !== "ready") { + throw new Error(`Local site unavailable: ${localSite.status}`); + } + if (slot.boundaryQuality !== "exact") { + throw new Error("Ready local relation site requires an exact slot"); + } + return { + localSite, + replacementRange: Object.freeze({ + from: + slot.source.from + localSite.querySite.typedPathRange.from, + to: + slot.source.from + localSite.querySite.typedPathRange.to, + }), + statementOffset: slot.source.from, + }; +} + +function relationPath( + containers: readonly string[], + name: string, +): SqlCanonicalRelationPath { + return Object.freeze([ + ...containers.map((value) => + Object.freeze({ + quoted: false, + role: "schema" as const, + value, + }), + ), + Object.freeze({ + quoted: false, + role: "relation" as const, + value: name, + }), + ]); +} + +function catalogRelation(input: { + readonly completionPathStart?: number; + readonly containers?: readonly string[]; + readonly detail?: string; + readonly entityId: string; + readonly kind?: SqlCatalogRelationKind; + readonly match?: "equivalent" | "exact"; + readonly name: string; +}): SqlValidatedCatalogRelation { + const path = relationPath( + input.containers ?? [], + input.name, + ); + const completionPathStart = + input.completionPathStart ?? 0; + if ( + completionPathStart !== 0 && + completionPathStart !== path.length - 1 + ) { + throw new Error("Test helper supports full or relation-only completion"); + } + const completionPath = + completionPathStart === 0 + ? path + : relationPath([], input.name); + const relation = { + canonicalPath: path, + completionPath, + completionPathStart, + completionText: completionPath + .map((component) => component.value) + .join("."), + entityId: input.entityId, + matchQuality: input.match ?? "exact", + relationKind: input.kind ?? "table", + }; + return Object.freeze( + input.detail === undefined + ? relation + : { ...relation, detail: input.detail }, + ); +} + +function readyResponse( + relations: readonly SqlValidatedCatalogRelation[], + coverage: SqlCatalogReadyCoverage = Object.freeze({ + kind: "complete", + }), +): Extract< + SqlValidatedCatalogSearchResponse, + { readonly status: "ready" } +> { + return Object.freeze({ + coverage, + epoch: Object.freeze({ generation: 1, token: "one" }), + relations: Object.freeze([...relations]), + status: "ready", + }); +} + +function usable( + response: SqlValidatedCatalogSearchResponse, +): SqlComposableCatalogOutcome { + return Object.freeze({ + observation: "equal", + response, + status: "usable", + }); +} + +function compose( + marked: string, + catalogOutcome: SqlComposableCatalogOutcome, +): SqlCompletionList { + const local = markedLocalSite(marked); + return composeSqlRelationCompletion({ + catalogOutcome, + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + localSite: local.localSite, + providerId: + catalogOutcome === null ? null : "catalog", + remainingIntentLeaseMs: 25, + replacementRange: local.replacementRange, + statementOffset: local.statementOffset, + }).value; +} + +describe("relation completion composition", () => { + it("ranks CTEs before deterministic catalog tiers and shadows only unqualified insertions", () => { + const relations = [ + catalogRelation({ + entityId: "equivalent", + match: "equivalent", + name: "aardvark", + }), + catalogRelation({ + completionPathStart: 0, + containers: ["public"], + entityId: "qualified-users", + name: "users", + }), + catalogRelation({ + entityId: "view", + kind: "view", + name: "beta", + }), + catalogRelation({ + entityId: "shadowed-users", + name: "users", + }), + catalogRelation({ + entityId: "table", + name: "alpha", + }), + ]; + const value = compose( + "WITH zed AS (SELECT 1), users AS (SELECT 1) SELECT * FROM |", + usable(readyResponse(relations)), + ); + + expect( + value.items.map((item) => [ + item.relationKind, + item.label, + item.edit.insert, + ]), + ).toEqual([ + ["cte", "users", "users"], + ["cte", "zed", "zed"], + ["table", "alpha", "alpha"], + ["view", "beta", "beta"], + ["table", "users", "public.users"], + ["table", "aardvark", "aardvark"], + ]); + expect( + value.items.some( + (item) => + item.provenance.kind === "catalog" && + item.provenance.entityId === "shadowed-users", + ), + ).toBe(false); + expect(value.isIncomplete).toBe(false); + }); + + it("uses the mapped whole-path replacement range and preserves qualified catalog candidates", () => { + const local = markedLocalSite( + "WITH users AS (SELECT 1) SELECT * FROM public.us|", + ); + const result = composeSqlRelationCompletion({ + catalogOutcome: usable( + readyResponse([ + catalogRelation({ + completionPathStart: 1, + containers: ["public"], + entityId: "users", + name: "users", + }), + ]), + ), + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + localSite: local.localSite, + providerId: "catalog", + remainingIntentLeaseMs: 0, + replacementRange: local.replacementRange, + statementOffset: local.statementOffset, + }); + + expect(result.value.items).toHaveLength(1); + expect(result.value.items[0]?.edit).toEqual({ + from: local.replacementRange.from, + insert: "users", + to: local.replacementRange.to, + }); + }); + + it("reports soft loading, terminal catalog evidence, and coverage in a stable issue order", () => { + const loading = compose( + "SELECT * FROM |", + Object.freeze({ status: "loading" }), + ); + expect(loading).toMatchObject({ + isIncomplete: true, + issues: [ + { + reason: "catalog-loading", + remainingIntentLeaseMs: 25, + }, + ], + }); + + const partial = compose( + 'SELECT * FROM "unterminated|', + usable( + readyResponse( + [], + Object.freeze({ kind: "partial" }), + ), + ), + ); + expect(partial.issues.map((issue) => issue.reason)).toEqual([ + "query-site-recovery", + "catalog-partial", + ]); + + const paginated = compose( + "SELECT * FROM |", + usable( + readyResponse( + [], + Object.freeze({ + continuationToken: "secret", + kind: "paginated", + }), + ), + ), + ); + expect(paginated.issues).toEqual([ + { reason: "catalog-paginated" }, + ]); + expect(JSON.stringify(paginated)).not.toContain("secret"); + }); + + it.each([ + ["execution-timeout", "catalog-timeout", "execution-timeout"], + ["malformed-response", "catalog-malformed", "malformed-response"], + ["overloaded", "catalog-overloaded", "queue-overloaded"], + ["provider-failed", "catalog-failed", "provider-rejected"], + ["queue-timeout", "catalog-queue-timeout", "queue-timeout"], + ] as const)( + "maps %s without discarding local evidence", + (reason, issue, reportReason) => { + const local = markedLocalSite( + "WITH local_table AS (SELECT 1) SELECT * FROM |", + ); + const result = composeSqlRelationCompletion({ + catalogOutcome: Object.freeze({ + reason, + status: "unavailable", + }), + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + localSite: local.localSite, + providerId: "catalog", + remainingIntentLeaseMs: 0, + replacementRange: local.replacementRange, + statementOffset: local.statementOffset, + }); + + expect(result.value.items[0]?.label).toBe("local_table"); + expect(result.value.issues).toContainEqual({ reason: issue }); + expect(result.sources).toEqual([ + { + feature: "relation-catalog", + outcome: "unavailable", + providerId: "catalog", + reason: reportReason, + }, + ]); + }, + ); + + it("composes terminal loading and failed provider evidence", () => { + const loading = compose( + "SELECT * FROM |", + usable( + Object.freeze({ + epoch: Object.freeze({ + generation: 2, + token: "loading", + }), + status: "loading", + }), + ), + ); + expect(loading.issues).toEqual([ + { + reason: "catalog-loading", + remainingIntentLeaseMs: 25, + }, + ]); + + const local = markedLocalSite("SELECT * FROM |"); + const failed = composeSqlRelationCompletion({ + catalogOutcome: usable( + Object.freeze({ + code: "authentication", + epoch: Object.freeze({ + generation: 2, + token: "failed", + }), + retry: "after-invalidation", + status: "failed", + }), + ), + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + localSite: local.localSite, + providerId: "catalog", + remainingIntentLeaseMs: 0, + replacementRange: local.replacementRange, + statementOffset: local.statementOffset, + }); + expect(failed.value.issues).toEqual([ + { reason: "catalog-failed" }, + ]); + expect(failed.sources).toEqual([ + { + code: "authentication", + feature: "relation-catalog", + outcome: "failed", + providerId: "catalog", + retry: "after-invalidation", + }, + ]); + }); + + it("fails closed for uncertain and hostile CTE prefix policy", () => { + const local = markedLocalSite( + "WITH local_table AS (SELECT 1) SELECT * FROM |", + ); + const complete = ( + cteIdentifierMatchesPrefix: + SqlRelationCompletionDialectRuntime["cteIdentifierMatchesPrefix"], + ): SqlCompletionList => { + const completion = Object.freeze({ + ...POSTGRESQL_SQL_RELATION_DIALECT.completion, + cteIdentifierMatchesPrefix, + }); + const dialect: SqlRelationDialectRuntime = Object.freeze({ + ...POSTGRESQL_SQL_RELATION_DIALECT, + completion, + }); + return composeSqlRelationCompletion({ + catalogOutcome: null, + dialect, + localSite: local.localSite, + providerId: null, + remainingIntentLeaseMs: 0, + replacementRange: local.replacementRange, + statementOffset: local.statementOffset, + }).value; + }; + + const unknown = complete(() => "unknown"); + expect(unknown.items).toEqual([]); + expect(unknown.issues).toContainEqual({ + reason: "cte-scope-uncertainty", + }); + + const throwing = complete(() => { + throw new Error("hostile prefix policy"); + }); + expect(throwing.items).toEqual([]); + expect(throwing.issues).toContainEqual({ + reason: "cte-scope-uncertainty", + }); + + const noMatch = complete(() => "no-match"); + expect(noMatch.items).toEqual([]); + expect(noMatch.isIncomplete).toBe(false); + }); + + it("retains catalog evidence when CTE shadow comparison is uncertain", () => { + const local = markedLocalSite( + "WITH users AS (SELECT 1) SELECT * FROM |", + ); + const completion = Object.freeze({ + ...POSTGRESQL_SQL_RELATION_DIALECT.completion, + compareCteIdentifiers: () => { + throw new Error("hostile comparison policy"); + }, + renderRelationPath: () => + Object.freeze({ + reason: "illegal-role-sequence" as const, + status: "unsupported" as const, + }), + }); + const dialect: SqlRelationDialectRuntime = Object.freeze({ + ...POSTGRESQL_SQL_RELATION_DIALECT, + completion, + }); + const result = composeSqlRelationCompletion({ + catalogOutcome: usable( + readyResponse([ + catalogRelation({ + detail: "A catalog relation", + entityId: "users", + name: "users", + }), + ]), + ), + dialect, + localSite: local.localSite, + providerId: "catalog", + remainingIntentLeaseMs: 0, + replacementRange: local.replacementRange, + statementOffset: local.statementOffset, + }); + + expect(result.value.items).toHaveLength(2); + expect(result.value.items[1]).toMatchObject({ + detail: "A catalog relation", + label: "users", + provenance: { entityId: "users" }, + }); + expect(result.value.issues).toContainEqual({ + reason: "cte-scope-uncertainty", + }); + + if (local.localSite.local.kind !== "unqualified") { + throw new Error("Expected an unqualified local site"); + } + const uncertainSite: typeof local.localSite = Object.freeze({ + ...local.localSite, + local: Object.freeze({ + ...local.localSite.local, + cteVisibility: Object.freeze({ + ...local.localSite.local.cteVisibility, + quality: "recovered" as const, + shadowing: Object.freeze({ + coverage: "unknown" as const, + }), + }), + }), + }); + const unknownCoverage = composeSqlRelationCompletion({ + catalogOutcome: usable( + readyResponse([ + catalogRelation({ + entityId: "users", + name: "users", + }), + ]), + ), + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + localSite: uncertainSite, + providerId: "catalog", + remainingIntentLeaseMs: 0, + replacementRange: local.replacementRange, + statementOffset: local.statementOffset, + }); + expect(unknownCoverage.value.items).toHaveLength(2); + expect(unknownCoverage.value.issues).toContainEqual({ + reason: "cte-scope-uncertainty", + }); + }); + + it("surfaces recursive CTE and opaque query-site recovery", () => { + const recursive = compose( + "WITH RECURSIVE local_table AS (SELECT * FROM |) SELECT * FROM local_table", + null, + ); + expect(recursive.issues.map((issue) => issue.reason)).toEqual([ + "cte-scope-uncertainty", + "recursive-cte-uncertainty", + ]); + + const local = markedLocalSite("SELECT * FROM |"); + const opaqueSite: typeof local.localSite = Object.freeze({ + ...local.localSite, + querySite: Object.freeze({ + ...local.localSite.querySite, + recognition: Object.freeze({ + issues: Object.freeze([ + "opaque-template-context", + ] as const), + quality: "recovered" as const, + }), + }), + }); + const opaque = composeSqlRelationCompletion({ + catalogOutcome: null, + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + localSite: opaqueSite, + providerId: null, + remainingIntentLeaseMs: 0, + replacementRange: local.replacementRange, + statementOffset: local.statementOffset, + }).value; + expect(opaque.issues).toContainEqual({ + reason: "opaque-template-context", + }); + }); + + it("uses path and entity IDs as deterministic final catalog ties", () => { + const relations = [ + catalogRelation({ + completionPathStart: 0, + containers: ["zeta"], + entityId: "z-last", + name: "users", + }), + catalogRelation({ + completionPathStart: 0, + containers: ["alpha"], + entityId: "b", + name: "users", + }), + catalogRelation({ + completionPathStart: 0, + containers: ["alpha"], + entityId: "a", + name: "users", + }), + ]; + const value = compose( + "SELECT * FROM |", + usable(readyResponse(relations)), + ); + expect( + value.items.map( + (item) => + item.provenance.kind === "catalog" + ? item.provenance.entityId + : "cte", + ), + ).toEqual(["a", "b", "z-last"]); + }); + + it("applies the result cap after ranking and returns deeply frozen output", () => { + const declarations = Array.from( + { length: MAX_RELATION_COMPLETION_RESULTS + 1 }, + (_, index) => `cte_${String(index).padStart(3, "0")} AS (SELECT 1)`, + ).join(", "); + const value = compose( + `WITH ${declarations} SELECT * FROM |`, + null, + ); + + expect(value.items).toHaveLength( + MAX_RELATION_COMPLETION_RESULTS, + ); + expect(value.items[0]?.label).toBe("cte_000"); + expect(value.items.at(-1)?.label).toBe("cte_099"); + expect(value.issues).toContainEqual({ reason: "result-limit" }); + expect(Object.isFrozen(value)).toBe(true); + expect(Object.isFrozen(value.items)).toBe(true); + expect(Object.isFrozen(value.items[0]?.edit)).toBe(true); + expect(Object.isFrozen(value.items[0]?.provenance)).toBe(true); + }); +}); diff --git a/src/vnext/__tests__/session-completion.bench.ts b/src/vnext/__tests__/session-completion.bench.ts new file mode 100644 index 0000000..b1b2739 --- /dev/null +++ b/src/vnext/__tests__/session-completion.bench.ts @@ -0,0 +1,152 @@ +import { bench, describe } from "vitest"; +import { + createSqlLanguageService, + duckdbDialect, +} from "../index.js"; +import type { + SqlCatalogRelation, +} from "../relation-completion-types.js"; +import type { + SqlDocumentContext, + SqlDocumentSession, + SqlLanguageService, +} from "../types.js"; + +interface BenchmarkContext extends SqlDocumentContext { + readonly engine: "benchmark"; +} + +const dialect = duckdbDialect(); +const tenKibibytes = 10 * 1_024; +const prefix = "WITH local_cte AS (SELECT 1) SELECT "; +const suffix = " FROM local"; +const projectionLength = + tenKibibytes - prefix.length - suffix.length; +const projection = `${"x,".repeat( + Math.floor((projectionLength - 1) / 2), +)}x`; +const tenKibibyteText = `${prefix}${projection}${" ".repeat( + projectionLength - projection.length, +)}${suffix}`; +if (tenKibibyteText.length !== tenKibibytes) { + throw new Error("Session completion benchmark must be exactly 10 KiB"); +} + +function openLocalSession( + service: SqlLanguageService, +): SqlDocumentSession { + return service.openDocument({ + context: { dialect: "duckdb", engine: "benchmark" }, + text: tenKibibyteText, + }); +} + +const localService = + createSqlLanguageService({ + dialects: [dialect], + }); +const warmLocalSession = openLocalSession(localService); +const warmPreflight = await warmLocalSession.complete({ + position: tenKibibyteText.length, + trigger: { kind: "invoked" }, +}); +if ( + warmPreflight.status !== "ready" || + warmPreflight.value.items.length !== 1 +) { + throw new Error("Warm session benchmark preflight failed"); +} + +const catalogRelations = Array.from( + { length: 100 }, + (_, index): SqlCatalogRelation => ({ + canonicalPath: [ + { + quoted: false, + role: "relation" as const, + value: `relation_${index}`, + }, + ], + completionPathStart: 0, + entityId: `relation-${index}`, + matchQuality: "exact" as const, + relationKind: "table" as const, + }), +); + +function createCatalogService(): SqlLanguageService { + return createSqlLanguageService({ + catalog: { + id: "benchmark-catalog", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: catalogRelations, + status: "ready", + }), + }, + dialects: [dialect], + }); +} + +const cachedCatalogService = createCatalogService(); +const cachedCatalogSession = cachedCatalogService.openDocument({ + context: { + catalog: { scope: "benchmark:cached" }, + dialect: "duckdb", + engine: "benchmark", + }, + text: "SELECT * FROM relation_", +}); +const cachedPreflight = await cachedCatalogSession.complete({ + position: 23, + trigger: { kind: "invoked" }, +}); +if ( + cachedPreflight.status !== "ready" || + cachedPreflight.value.items.length !== 100 +) { + throw new Error("Cached catalog benchmark preflight failed"); +} + +describe("session completion", () => { + bench("cold 10 KiB local completion", async () => { + const session = openLocalSession(localService); + await session.complete({ + position: tenKibibyteText.length, + trigger: { kind: "invoked" }, + }); + session.dispose(); + }); + + bench("warm 10 KiB local completion", async () => { + await warmLocalSession.complete({ + position: tenKibibyteText.length, + trigger: { kind: "invoked" }, + }); + }); + + bench("cached 100-candidate catalog completion", async () => { + await cachedCatalogSession.complete({ + position: 23, + trigger: { kind: "invoked" }, + }); + }); + + bench("cold 100-candidate catalog completion", async () => { + const service = createCatalogService(); + const session = service.openDocument({ + context: { + catalog: { scope: "benchmark:cold" }, + dialect: "duckdb", + engine: "benchmark", + }, + text: "SELECT * FROM relation_", + }); + await session.complete({ + position: 23, + trigger: { kind: "invoked" }, + }); + service.dispose(); + }); +}); diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index ed581dd..130b5c4 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { bigQueryDialect, createSqlLanguageService, @@ -27,7 +27,13 @@ import { import type { SqlDocumentContext, SqlDocumentReplacement, + SqlRevision, } from "../types.js"; +import type { + SqlCatalogInvalidation, + SqlRelationCatalogProvider, + SqlSessionChangeEvent, +} from "../relation-completion-types.js"; interface TestContext extends SqlDocumentContext { readonly engine: string; @@ -148,148 +154,970 @@ describe("dialect definitions", () => { }); }); -describe("statement-index session cache", () => { - it("binds lexical behavior through authentic dialect handles", () => { - const service = new DefaultSqlLanguageService({ - dialects: [ - bigQueryDialect(), - dremioDialect(), - duckdb, - postgres, - ], +describe("relation completion session integration", () => { + function catalogProvider( + search: SqlRelationCatalogProvider["search"], + subscribe?: SqlRelationCatalogProvider["subscribe"], + ): SqlRelationCatalogProvider { + return subscribe + ? { id: "test-catalog", search, subscribe } + : { id: "test-catalog", search }; + } + + it("completes visible CTEs without a catalog provider", async () => { + const { service, session } = openSession( + "WITH source_data AS (SELECT 1) SELECT * FROM sou", + ); + const result = await session.complete({ + position: 48, + trigger: { kind: "invoked" }, + }); + expect(result).toMatchObject({ + status: "ready", + value: { + items: [ + { + edit: { from: 45, insert: "source_data", to: 48 }, + label: "source_data", + relationKind: "cte", + }, + ], + }, }); - const cases = [ - { - dialect: "bigquery", - profile: BIGQUERY_SQL_LEXICAL_PROFILE, - text: "# hidden; still hidden\nSELECT r'''a;b''';", + service.dispose(); + }); + + it("composes a validated catalog page through the public session", async () => { + const service = createSqlLanguageService({ + catalog: catalogProvider(async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [ + { + canonicalPath: [ + { + quoted: false, + role: "catalog", + value: "memory", + }, + { + quoted: false, + role: "schema", + value: "main", + }, + { + quoted: false, + role: "relation", + value: "users", + }, + ], + completionPathStart: 2, + entityId: "users", + matchQuality: "exact", + relationKind: "table", + }, + ], + status: "ready", + })), + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", }, - { - dialect: "dremio", - profile: DREMIO_SQL_LEXICAL_PROFILE, - text: "# code; SELECT $$a;b$$;", + text: "SELECT * FROM us", + }); + await expect( + session.complete({ + position: 16, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + sources: [ + { + coverage: "complete", + outcome: "ready", + providerId: "test-catalog", + }, + ], + status: "ready", + value: { + isIncomplete: false, + items: [ + { + edit: { from: 14, insert: "users", to: 16 }, + label: "users", + provenance: { + entityId: "users", + providerId: "test-catalog", + }, + }, + ], }, - { + }); + service.dispose(); + }); + + it("keeps completion edits in absolute UTF-16 document coordinates", async () => { + const service = createSqlLanguageService({ + catalog: catalogProvider(async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [ + { + canonicalPath: [ + { + quoted: false, + role: "relation", + value: "users", + }, + ], + completionPathStart: 0, + entityId: "users", + matchQuality: "exact", + relationKind: "table", + }, + ], + status: "ready", + })), + dialects: [duckdb], + }); + const text = "SELECT '😀'; SELECT * FROM us"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, dialect: "duckdb", - profile: DUCKDB_SQL_LEXICAL_PROFILE, - text: "SELECT $$a;b$$; /* outer /* inner; */ done */", + engine: "local", }, - { - dialect: "postgresql", - profile: POSTGRESQL_SQL_LEXICAL_PROFILE, - text: "SELECT E'a\\';b'; SELECT $tag$c;d$tag$;", + text, + }); + const result = await session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }); + expect(result).toMatchObject({ + status: "ready", + value: { + items: [ + { + edit: { + from: text.length - 2, + insert: "users", + to: text.length, + }, + kind: "relation", + }, + ], }, - ] as const; - - for (const testCase of cases) { - const session = service.openDocument({ - context: { - dialect: testCase.dialect, - engine: "warehouse", - }, - text: testCase.text, - }); - expect(session.getStatementIndexForTesting()).toEqual( - buildSqlStatementIndex(testCase.text, testCase.profile), - ); - } + }); + service.dispose(); }); - it("builds lazily and reuses the current cache", () => { - const { session } = openSession("SELECT 1; SELECT 2"); - expect(session.cachedStatementIndexForTesting).toBeNull(); - - const index = session.getStatementIndexForTesting(); - expect(session.cachedStatementIndexForTesting).toBe(index); - expect(session.getStatementIndexForTesting()).toBe(index); + it("uses absolute coordinates after a masked embedded region", async () => { + const service = createSqlLanguageService({ + catalog: catalogProvider(async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [ + { + canonicalPath: [ + { + quoted: false, + role: "relation", + value: "users", + }, + ], + completionPathStart: 0, + entityId: "users", + matchQuality: "exact", + relationKind: "table", + }, + ], + status: "ready", + })), + dialects: [duckdb], + }); + const text = "SELECT * FROM {df}; SELECT * FROM us"; + const embeddedFrom = text.indexOf("{df}"); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + embeddedRegions: [ + { + from: embeddedFrom, + language: "python", + to: embeddedFrom + 4, + }, + ], + text, + }); + const result = await session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }); + expect(result).toMatchObject({ + status: "ready", + value: { + items: [ + { + edit: { + from: text.length - 2, + insert: "users", + to: text.length, + }, + }, + ], + }, + }); + service.dispose(); }); - it("retains the index for unrelated context changes", () => { - const { session } = openSession("SELECT 1; SELECT 2"); - const index = session.getStatementIndexForTesting(); - session.update({ - baseRevision: session.revision, - context: { dialect: "duckdb", engine: "remote" }, + it("reports terminal loading with a bounded completion intent", async () => { + const service = createSqlLanguageService({ + catalog: catalogProvider(async () => ({ + epoch: { generation: 0, token: "loading" }, + status: "loading", + })), + dialects: [duckdb], }); - expect(session.cachedStatementIndexForTesting).toBe(index); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const result = await session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + expect(result).toMatchObject({ + status: "ready", + value: { + isIncomplete: true, + issues: [ + { + reason: "catalog-loading", + remainingIntentLeaseMs: + 1_000, + }, + ], + }, + }); + service.dispose(); }); - it("invalidates the index when lexical profile identity changes", () => { - const { session } = openSession("SELECT $$a;b$$;"); - const index = session.getStatementIndexForTesting(); - session.update({ - baseRevision: session.revision, - context: { dialect: "postgresql", engine: "warehouse" }, + it("returns on the soft budget and emits one availability revision", async () => { + let resolveSearch: + | (( + response: Awaited< + ReturnType + >, + ) => void) + | undefined; + const searchResult = new Promise< + Awaited> + >((resolve) => { + resolveSearch = resolve; + }); + const service = createSqlLanguageService({ + catalog: catalogProvider(() => searchResult), + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], }); - expect(session.cachedStatementIndexForTesting).toBeNull(); - expect(session.getStatementIndexForTesting()).not.toBe(index); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const events: string[] = []; + session.onDidChange((event) => { + events.push(event.reason); + }); + const result = await session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + expect(result).toMatchObject({ + status: "ready", + value: { + issues: [{ reason: "catalog-loading" }], + }, + }); + resolveSearch?.({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(events).toEqual(["catalog-availability"]); + service.dispose(); }); - it("reuses the index across no-op document mutations", () => { - const { session } = openSession("SELECT 1"); - const initialSource = session.snapshotForTesting.source; - const index = session.getStatementIndexForTesting(); - - session.update({ - embeddedRegions: [], - baseRevision: session.revision, - document: { kind: "changes", changes: [] }, + it("supersedes an older same-key request without restarting work", async () => { + let searchCount = 0; + let resolveSearch: + | (( + response: Awaited< + ReturnType + >, + ) => void) + | undefined; + const searchResult = new Promise< + Awaited> + >((resolve) => { + resolveSearch = resolve; + }); + const service = createSqlLanguageService({ + catalog: catalogProvider(() => { + searchCount += 1; + return searchResult; + }), + dialects: [duckdb], }); - expect(session.snapshotForTesting.source).toBe(initialSource); - expect(session.cachedStatementIndexForTesting).toBe(index); - - session.update({ - embeddedRegions: [], - baseRevision: session.revision, - document: { kind: "replace", text: "SELECT 1" }, + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", }); - expect(session.cachedStatementIndexForTesting).toBe(index); + const first = session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + await Promise.resolve(); + const second = session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + resolveSearch?.({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }); + await expect(first).resolves.toMatchObject({ + reason: "superseded", + status: "cancelled", + }); + await expect(second).resolves.toMatchObject({ + status: "ready", + }); + expect(searchCount).toBe(1); + service.dispose(); }); - it("updates incrementally to the full-scan oracle", () => { - const { session } = openSession("SELECT 1; SELECT 2; SELECT 3"); - const previous = session.getStatementIndexForTesting(); - session.update({ - embeddedRegions: [], - baseRevision: session.revision, - document: { - kind: "changes", - changes: [{ from: 17, insert: "20", to: 18 }], + it("does not supersede valid work for a malformed request", async () => { + let resolveSearch: + | (( + response: Awaited< + ReturnType + >, + ) => void) + | undefined; + const searchResult = new Promise< + Awaited> + >((resolve) => { + resolveSearch = resolve; + }); + const service = createSqlLanguageService({ + catalog: catalogProvider(() => searchResult), + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", }, + text: "SELECT * FROM ", + }); + const valid = session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + await expect( + session.complete({ + position: 14, + trigger: { + character: ".", + kind: "invoked", + }, + } as never), + ).rejects.toMatchObject({ + code: "invalid-completion-request", + }); + resolveSearch?.({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }); + await expect(valid).resolves.toMatchObject({ + status: "ready", }); + service.dispose(); + }); - const cached = session.cachedStatementIndexForTesting; - expect(cached).not.toBeNull(); - expect(cached).toEqual( - buildSqlStatementIndex( - "SELECT 1; SELECT 20; SELECT 3", - DUCKDB_SQL_LEXICAL_PROFILE, + it("supersedes pending work when a higher catalog epoch arrives", async () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = createSqlLanguageService({ + catalog: catalogProvider( + () => + new Promise(() => { + // Intentionally unsettled; invalidation owns cancellation. + }), + (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, ), - ); - expect(cached).not.toBe(previous); - expect(cached?.slots[0]).toBe(previous.slots[0]); + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const events: string[] = []; + session.onDidChange((event) => { + events.push(event.reason); + }); + const result = session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + invalidate?.({ + epoch: { generation: 1, token: "changed" }, + }); + await expect(result).resolves.toMatchObject({ + reason: "superseded", + status: "cancelled", + }); + expect(events).toEqual(["catalog"]); + service.dispose(); }); - it("clears changed replacements and preserves the cache on failure", () => { - const { session } = openSession("SELECT 1"); - const index = session.getStatementIndexForTesting(); - const revision = session.revision; - expectSessionError("stale-revision", () => { - session.update({ - embeddedRegions: [], - baseRevision: {} as never, - document: { kind: "replace", text: "SELECT 2" }, + it("settles pending completion on caller abort, update, and disposal", async () => { + const createPendingSession = () => { + const service = createSqlLanguageService({ + catalog: catalogProvider( + () => + new Promise(() => { + // Intentionally unsettled; session cancellation owns completion. + }), + ), + dialects: [duckdb], }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + return { service, session }; + }; + + const caller = createPendingSession(); + const controller = new AbortController(); + const callerResult = caller.session.complete({ + position: 14, + signal: controller.signal, + trigger: { kind: "invoked" }, + }); + controller.abort(); + await expect(callerResult).resolves.toMatchObject({ + reason: "caller", + status: "cancelled", + }); + caller.service.dispose(); + + const updated = createPendingSession(); + const updatedResult = updated.session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + updated.session.update({ + baseRevision: updated.session.revision, + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "remote", + }, }); - expect(session.revision).toBe(revision); - expect(session.cachedStatementIndexForTesting).toBe(index); + await expect(updatedResult).resolves.toMatchObject({ + reason: "superseded", + status: "cancelled", + }); + updated.service.dispose(); + + const disposed = createPendingSession(); + const disposedResult = disposed.session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + disposed.session.dispose(); + await expect(disposedResult).resolves.toMatchObject({ + reason: "disposed", + status: "cancelled", + }); + disposed.service.dispose(); + }); + it("replaces catalog ownership when the scope changes", async () => { + const scopes: string[] = []; + const service = createSqlLanguageService({ + catalog: catalogProvider(async (request) => { + scopes.push(request.scope); + return { + coverage: { kind: "complete" }, + epoch: { generation: 0, token: request.scope }, + relations: [], + status: "ready", + }; + }), + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); session.update({ - embeddedRegions: [], baseRevision: session.revision, - document: { kind: "replace", text: "SELECT 2" }, + context: { + catalog: { scope: "connection:2" }, + dialect: "duckdb", + engine: "local", + }, }); - expect(session.cachedStatementIndexForTesting).toBeNull(); + await session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + expect(scopes).toEqual(["connection:1", "connection:2"]); + service.dispose(); + }); + + it("advances revision and isolates listeners on catalog invalidation", () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = createSqlLanguageService({ + catalog: catalogProvider( + async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }), + (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + ), + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const previous = session.revision; + const events: string[] = []; + session.onDidChange((event) => { + events.push(event.reason); + throw new Error("isolated"); + }); + session.onDidChange((event) => { + events.push(`${event.reason}:second`); + }); + invalidate?.({ + epoch: { generation: 1, token: "changed" }, + }); + expect(session.revision).not.toBe(previous); + expect(events).toEqual(["catalog", "catalog:second"]); + service.dispose(); + }); + + it("stops a stale catalog event after a listener updates the session", () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = createSqlLanguageService({ + catalog: catalogProvider( + async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }), + (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + ), + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const events: SqlRevision[] = []; + session.onDidChange((event) => { + events.push(event.revision); + session.update({ + baseRevision: event.revision, + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "remote", + }, + }); + }); + session.onDidChange((event) => { + events.push(event.revision); + }); + + invalidate?.({ + epoch: { generation: 1, token: "changed" }, + }); + + expect(events).toHaveLength(1); + expect(session.revision).not.toBe(events[0]); + service.dispose(); + }); + + it("does not deliver an outer catalog event after nested invalidation", () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = createSqlLanguageService({ + catalog: catalogProvider( + async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }), + (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + ), + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const events: SqlRevision[] = []; + let nested = false; + session.onDidChange((event) => { + events.push(event.revision); + if (!nested) { + nested = true; + invalidate?.({ + epoch: { generation: 2, token: "nested" }, + }); + } + }); + session.onDidChange((event) => { + events.push(event.revision); + }); + + invalidate?.({ + epoch: { generation: 1, token: "outer" }, + }); + + expect(events).toHaveLength(4); + expect(events[0]).toBe(events[1]); + expect(events[1]).not.toBe(events[2]); + expect(events[2]).toBe(events[3]); + expect(session.revision).toBe(events[2]); + service.dispose(); + }); + + it("keeps duplicate listener subscriptions independent", () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = createSqlLanguageService({ + catalog: catalogProvider( + async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }), + (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + ), + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const events: string[] = []; + const listener = (event: SqlSessionChangeEvent): void => { + events.push(event.reason); + }; + const first = session.onDidChange(listener); + const second = session.onDidChange(listener); + + first.dispose(); + invalidate?.({ + epoch: { generation: 1, token: "first" }, + }); + expect(events).toEqual(["catalog"]); + + second.dispose(); + invalidate?.({ + epoch: { generation: 2, token: "second" }, + }); + expect(events).toEqual(["catalog"]); + service.dispose(); + }); + + it("validates completion configuration and request input", async () => { + expectSessionError("invalid-service-options", () => { + createSqlLanguageService({ + completion: { catalogResponseBudgetMs: 51 }, + dialects: [duckdb], + }); + }); + const { service, session } = openSession(); + await expect( + session.complete(null as never), + ).rejects.toMatchObject({ + code: "invalid-completion-request", + }); + service.dispose(); + }); + + it("rejects catalog context without a configured provider atomically", () => { + const service = createSqlLanguageService({ + dialects: [duckdb], + }); + expectSessionError("invalid-context", () => { + service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + }); + const session = service.openDocument({ + context: { + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const revision = session.revision; + expectSessionError("invalid-context", () => { + session.update({ + baseRevision: revision, + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + }); + }); + expect(session.revision).toBe(revision); + service.dispose(); + }); +}); + +describe("statement-index session cache", () => { + it("binds lexical behavior through authentic dialect handles", () => { + const service = new DefaultSqlLanguageService({ + dialects: [ + bigQueryDialect(), + dremioDialect(), + duckdb, + postgres, + ], + }); + const cases = [ + { + dialect: "bigquery", + profile: BIGQUERY_SQL_LEXICAL_PROFILE, + text: "# hidden; still hidden\nSELECT r'''a;b''';", + }, + { + dialect: "dremio", + profile: DREMIO_SQL_LEXICAL_PROFILE, + text: "# code; SELECT $$a;b$$;", + }, + { + dialect: "duckdb", + profile: DUCKDB_SQL_LEXICAL_PROFILE, + text: "SELECT $$a;b$$; /* outer /* inner; */ done */", + }, + { + dialect: "postgresql", + profile: POSTGRESQL_SQL_LEXICAL_PROFILE, + text: "SELECT E'a\\';b'; SELECT $tag$c;d$tag$;", + }, + ] as const; + + for (const testCase of cases) { + const session = service.openDocument({ + context: { + dialect: testCase.dialect, + engine: "warehouse", + }, + text: testCase.text, + }); + expect(session.getStatementIndexForTesting()).toEqual( + buildSqlStatementIndex(testCase.text, testCase.profile), + ); + } + }); + + it("builds lazily and reuses the current cache", () => { + const { session } = openSession("SELECT 1; SELECT 2"); + expect(session.cachedStatementIndexForTesting).toBeNull(); + + const index = session.getStatementIndexForTesting(); + expect(session.cachedStatementIndexForTesting).toBe(index); + expect(session.getStatementIndexForTesting()).toBe(index); + }); + + it("retains the index for unrelated context changes", () => { + const { session } = openSession("SELECT 1; SELECT 2"); + const index = session.getStatementIndexForTesting(); + session.update({ + baseRevision: session.revision, + context: { dialect: "duckdb", engine: "remote" }, + }); + expect(session.cachedStatementIndexForTesting).toBe(index); + }); + + it("invalidates the index when lexical profile identity changes", () => { + const { session } = openSession("SELECT $$a;b$$;"); + const index = session.getStatementIndexForTesting(); + session.update({ + baseRevision: session.revision, + context: { dialect: "postgresql", engine: "warehouse" }, + }); + expect(session.cachedStatementIndexForTesting).toBeNull(); + expect(session.getStatementIndexForTesting()).not.toBe(index); + }); + + it("reuses the index across no-op document mutations", () => { + const { session } = openSession("SELECT 1"); + const initialSource = session.snapshotForTesting.source; + const index = session.getStatementIndexForTesting(); + + session.update({ + embeddedRegions: [], + baseRevision: session.revision, + document: { kind: "changes", changes: [] }, + }); + expect(session.snapshotForTesting.source).toBe(initialSource); + expect(session.cachedStatementIndexForTesting).toBe(index); + + session.update({ + embeddedRegions: [], + baseRevision: session.revision, + document: { kind: "replace", text: "SELECT 1" }, + }); + expect(session.cachedStatementIndexForTesting).toBe(index); + }); + + it("updates incrementally to the full-scan oracle", () => { + const { session } = openSession("SELECT 1; SELECT 2; SELECT 3"); + const previous = session.getStatementIndexForTesting(); + session.update({ + embeddedRegions: [], + baseRevision: session.revision, + document: { + kind: "changes", + changes: [{ from: 17, insert: "20", to: 18 }], + }, + }); + + const cached = session.cachedStatementIndexForTesting; + expect(cached).not.toBeNull(); + expect(cached).toEqual( + buildSqlStatementIndex( + "SELECT 1; SELECT 20; SELECT 3", + DUCKDB_SQL_LEXICAL_PROFILE, + ), + ); + expect(cached).not.toBe(previous); + expect(cached?.slots[0]).toBe(previous.slots[0]); + }); + + it("clears changed replacements and preserves the cache on failure", () => { + const { session } = openSession("SELECT 1"); + const index = session.getStatementIndexForTesting(); + const revision = session.revision; + expectSessionError("stale-revision", () => { + session.update({ + embeddedRegions: [], + baseRevision: {} as never, + document: { kind: "replace", text: "SELECT 2" }, + }); + }); + expect(session.revision).toBe(revision); + expect(session.cachedStatementIndexForTesting).toBe(index); + + session.update({ + embeddedRegions: [], + baseRevision: session.revision, + document: { kind: "replace", text: "SELECT 2" }, + }); + expect(session.cachedStatementIndexForTesting).toBeNull(); }); it("releases the cache on disposal", () => { @@ -1937,3 +2765,1507 @@ describe("lifecycle", () => { }); }); }); + +describe("session coverage hardening", () => { + const readyCatalog: SqlRelationCatalogProvider = { + id: "coverage-catalog", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }), + }; + + function catalogService( + provider: SqlRelationCatalogProvider = readyCatalog, + ) { + return createSqlLanguageService({ + catalog: provider, + dialects: [duckdb, postgres], + }); + } + + it.each([ + { scope: "" }, + { scope: "x".repeat(513) }, + { scope: 1 }, + { scope: "valid", searchPath: "main" }, + { + scope: "valid", + searchPath: Array.from({ length: 33 }, () => [ + { quoted: false, value: "main" }, + ]), + }, + { scope: "valid", searchPath: ["main"] }, + { scope: "valid", searchPath: [[]] }, + { + scope: "valid", + searchPath: [ + Array.from({ length: 5 }, () => ({ + quoted: false, + value: "main", + })), + ], + }, + { scope: "valid", searchPath: [[null]] }, + { + scope: "valid", + searchPath: [[{ quoted: false, value: 1 }]], + }, + { + scope: "valid", + searchPath: [[{ quoted: false, value: "" }]], + }, + { + scope: "valid", + searchPath: [ + [{ quoted: false, value: "x".repeat(257) }], + ], + }, + { + scope: "valid", + searchPath: [[{ quoted: "no", value: "main" }]], + }, + ])("rejects invalid catalog context %# atomically", (catalog) => { + const service = catalogService(); + expectSessionError("invalid-context", () => { + service.openDocument({ + context: { + catalog, + dialect: "duckdb", + engine: "local", + } as never, + text: "SELECT 1", + }); + }); + service.dispose(); + }); + + it.each([ + { position: Number.NaN, trigger: { kind: "invoked" } }, + { position: -1, trigger: { kind: "invoked" } }, + { position: 9, trigger: { kind: "invoked" } }, + { position: 0, trigger: null }, + { position: 0, trigger: { kind: "unknown" } }, + { + position: 0, + trigger: { character: 1, kind: "trigger-character" }, + }, + { + position: 0, + trigger: { character: "", kind: "trigger-character" }, + }, + { + position: 0, + trigger: { character: "ab", kind: "trigger-character" }, + }, + { + position: 0, + trigger: { character: "😀x", kind: "trigger-character" }, + }, + { + position: 0, + signal: {}, + trigger: { kind: "invoked" }, + }, + ])("rejects malformed completion request %#", async (request) => { + const { service, session } = openSession("SELECT 1"); + await expect( + session.complete(request as never), + ).rejects.toMatchObject({ + code: "invalid-completion-request", + }); + service.dispose(); + }); + + it("accepts a one-code-point trigger and rejects listeners after disposal", async () => { + const { service, session } = openSession( + "SELECT * FROM ", + ); + await expect( + session.complete({ + position: 14, + trigger: { character: "😀", kind: "trigger-character" }, + }), + ).resolves.toMatchObject({ status: "ready" }); + expectSessionError("invalid-completion-request", () => { + session.onDidChange(null as never); + }); + const subscription = session.onDidChange(() => {}); + subscription.dispose(); + subscription.dispose(); + session.dispose(); + expectSessionError("session-disposed", () => { + session.onDidChange(() => {}); + }); + await expect( + session.complete({ + position: 0, + trigger: { kind: "invoked" }, + }), + ).rejects.toMatchObject({ code: "session-disposed" }); + service.dispose(); + }); + + it.each([ + { + reason: "opaque-statement", + text: "DELIMITER $$", + }, + { + reason: "resource-limit", + text: `SELECT * FROM ${" ".repeat(65_537)}`, + }, + { + reason: "inactive", + text: "SELECT 1", + }, + ] as const)("returns $reason local unavailability", async ({ reason, text }) => { + const service = createSqlLanguageService({ + dialects: [postgres], + }); + const session = service.openDocument({ + context: { dialect: "postgresql", engine: "local" }, + text, + }); + await expect( + session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + reason, + retryable: false, + status: "unavailable", + }); + service.dispose(); + }); + + it.each([ + null, + "bad", + { catalogResponseBudgetMs: "fast" }, + { catalogResponseBudgetMs: Number.NaN }, + { catalogResponseBudgetMs: -1 }, + ])("rejects invalid completion options %#", (completion) => { + expectSessionError("invalid-service-options", () => { + createSqlLanguageService({ + completion, + dialects: [duckdb], + } as never); + }); + }); + + it("rejects a malformed catalog provider", () => { + expectSessionError("invalid-service-options", () => { + createSqlLanguageService({ + catalog: { id: "", search: async () => ({}) }, + dialects: [duckdb], + } as never); + }); + }); + + it.each([ + { + expectedIssue: "catalog-failed", + response: { + code: "unavailable", + epoch: { generation: 0, token: "failed" }, + retry: "next-request", + status: "failed", + }, + }, + { + expectedIssue: "catalog-malformed", + response: { + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "bad" }, + relations: [{ bad: true }], + status: "ready", + }, + }, + ])("maps catalog outcome $expectedIssue", async ({ + expectedIssue, + response, + }) => { + const service = catalogService({ + id: `catalog-${expectedIssue}`, + search: async () => response as never, + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:coverage" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await expect( + session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + status: "ready", + value: { + issues: [{ reason: expectedIssue }], + }, + }); + service.dispose(); + }); + + it("maps provider rejection and already-aborted requests", async () => { + const service = catalogService({ + id: "rejecting", + search: async () => { + throw new Error("private provider failure"); + }, + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:coverage" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await expect( + session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + status: "ready", + value: { issues: [{ reason: "catalog-failed" }] }, + }); + const controller = new AbortController(); + controller.abort(); + await expect( + session.complete({ + position: 14, + signal: controller.signal, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + reason: "caller", + status: "cancelled", + }); + service.dispose(); + }); + + it("expires terminal loading intent and clears it idempotently", async () => { + vi.useFakeTimers(); + let service: + | ReturnType> + | undefined; + try { + service = catalogService({ + id: "loading", + search: async () => ({ + epoch: { generation: 0, token: "loading" }, + status: "loading", + }), + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:coverage" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const revision = session.revision; + const events: string[] = []; + session.onDidChange((event) => { + events.push(event.reason); + }); + await expect(session.complete({ + position: 14, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + status: "ready", + value: { + issues: [ + { + reason: "catalog-loading", + remainingIntentLeaseMs: 1_000, + }, + ], + }, + }); + expect(vi.getTimerCount()).toBe(1); + await vi.advanceTimersByTimeAsync(999); + expect(vi.getTimerCount()).toBe(1); + expect(session.revision).toBe(revision); + expect(events).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + expect(vi.getTimerCount()).toBe(0); + expect(session.revision).toBe(revision); + expect(events).toEqual([]); + await vi.advanceTimersByTimeAsync(1_000); + session.update({ + baseRevision: session.revision, + context: { + catalog: { scope: "connection:coverage" }, + dialect: "duckdb", + engine: "changed", + }, + }); + expect(vi.getTimerCount()).toBe(0); + } finally { + service?.dispose(); + vi.useRealTimers(); + } + }); + + it("enforces context node and property aggregate limits", () => { + const service = createService(); + expectSessionError("invalid-context", () => { + service.openDocument({ + context: { + dialect: "duckdb", + engine: "local", + nodes: Array.from({ length: 10_001 }, () => ({})), + } as never, + text: "", + }); + }); + const manyProperties = Object.fromEntries( + Array.from({ length: 50_001 }, (_, index) => [ + `p${index}`, + null, + ]), + ); + expectSessionError("invalid-context", () => { + service.openDocument({ + context: { + dialect: "duckdb", + engine: "local", + manyProperties, + } as never, + text: "", + }); + }); + service.dispose(); + }); + + it("keeps listener subscriptions independently disposable during dispatch", () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = catalogService({ + ...readyCatalog, + subscribe: (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:listeners" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT 1", + }); + const calls: string[] = []; + let second = session.onDidChange(() => { + calls.push("second"); + }); + const first = session.onDidChange(() => { + calls.push("first"); + second.dispose(); + }); + second.dispose(); + second = session.onDidChange(() => { + calls.push("second"); + }); + invalidate?.({ + epoch: { generation: 1, token: "changed" }, + }); + expect(calls).toEqual(["first"]); + first.dispose(); + service.dispose(); + }); + + it.each(["update", "dispose"] as const)( + "releases a retained refresh intent on %s", + async (action) => { + let aborted = 0; + const service = createSqlLanguageService({ + catalog: { + id: `pending-${action}`, + search: async (_request, signal) => + new Promise((resolve) => { + signal.addEventListener( + "abort", + () => { + aborted += 1; + resolve({ + epoch: { generation: 0, token: "late" }, + status: "loading", + }); + }, + { once: true }, + ); + }), + }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: `connection:${action}` }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await expect( + session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + status: "ready", + value: { + issues: [{ reason: "catalog-loading" }], + }, + }); + if (action === "update") { + session.update({ + baseRevision: session.revision, + context: { + catalog: { scope: `connection:${action}` }, + dialect: "duckdb", + engine: "changed", + }, + }); + } else { + session.dispose(); + } + await Promise.resolve(); + expect(aborted).toBe(1); + service.dispose(); + }, + ); + + it("hands a retained intent to a different-key completion", async () => { + const aborted: string[] = []; + const service = createSqlLanguageService({ + catalog: { + id: "different-key", + search: async (request, signal) => + new Promise((resolve) => { + signal.addEventListener( + "abort", + () => { + aborted.push(request.prefix.value); + resolve({ + epoch: { generation: 0, token: "late" }, + status: "loading", + }); + }, + { once: true }, + ); + }), + }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const text = "SELECT * FROM a; SELECT * FROM b"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:different-key" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + await session.complete({ + position: text.indexOf("a;") + 1, + trigger: { kind: "invoked" }, + }); + await session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }); + await Promise.resolve(); + expect(aborted).toContain("a"); + service.dispose(); + }); + + it("maps synchronous provider budget exhaustion to catalog timeout", async () => { + const service = catalogService({ + id: "synchronous-timeout", + search: async () => { + const deadline = performance.now() + 10; + while (performance.now() < deadline) { + // Deliberately exceed the coordinator's synchronous budget. + } + return { + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "late" }, + relations: [], + status: "ready", + }; + }, + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:sync-timeout" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await expect( + session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + status: "ready", + value: { issues: [{ reason: "catalog-timeout" }] }, + }); + service.dispose(); + }); + + it("rejects non-string and unknown dialect context values", () => { + const service = createService(); + for (const dialect of [1, "unknown"]) { + expectSessionError("invalid-dialect", () => { + service.openDocument({ + context: { dialect, engine: "local" } as never, + text: "", + }); + }); + } + service.dispose(); + }); + + it.each([ + ["SELECT * FROM users ON ", "unsupported-query-site"], + [ + "SELECT * FROM users NATURAL WHERE ", + "ambiguous-query-site", + ], + ])("maps query-site uncertainty in %s", async (text, reason) => { + const service = createSqlLanguageService({ + dialects: [postgres], + }); + const session = service.openDocument({ + context: { dialect: "postgresql", engine: "local" }, + text, + }); + await expect( + session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + reason, + retryable: false, + status: "unavailable", + }); + service.dispose(); + }); + + it("cancels previous active work when the replacement site is inactive", async () => { + const service = createSqlLanguageService({ + catalog: { + id: "inactive-replacement", + search: async () => + new Promise(() => { + // Cancelled by the second completion. + }), + }, + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:inactive-replacement" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const first = session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + await Promise.resolve(); + await expect( + session.complete({ + position: 0, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + reason: "inactive", + status: "unavailable", + }); + await expect(first).resolves.toMatchObject({ + reason: "superseded", + status: "cancelled", + }); + service.dispose(); + }); + + it("cancels a retained intent when the replacement site is inactive", async () => { + let aborts = 0; + const service = createSqlLanguageService({ + catalog: { + id: "intent-inactive-replacement", + search: async (_request, signal) => + new Promise((resolve) => { + signal.addEventListener("abort", () => { + aborts += 1; + resolve({ + epoch: { generation: 0, token: "late" }, + status: "loading", + }); + }); + }), + }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { + scope: "connection:intent-inactive-replacement", + }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + await session.complete({ + position: 0, + trigger: { kind: "invoked" }, + }); + await Promise.resolve(); + expect(aborts).toBe(1); + service.dispose(); + }); + + it("preserves FIFO nested dispatch across shared-scope sessions", () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = catalogService({ + ...readyCatalog, + subscribe: (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + }); + const open = () => + service.openDocument({ + context: { + catalog: { scope: "connection:shared-dispatch" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT 1", + }); + const first = open(); + const second = open(); + const events: string[] = []; + first.onDidChange(() => { + events.push("first"); + if (events.length === 1) { + invalidate?.({ + epoch: { generation: 2, token: "nested" }, + }); + } + }); + second.onDidChange(() => { + events.push("second"); + }); + invalidate?.({ + epoch: { generation: 1, token: "outer" }, + }); + expect(events).toEqual([ + "first", + "second", + "first", + "second", + ]); + service.dispose(); + }); + + it("fails catalog ownership closed after live-scope capacity", async () => { + const service = catalogService(); + const sessions = Array.from({ length: 129 }, (_, index) => + service.openDocument({ + context: { + catalog: { scope: `connection:capacity:${index}` }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }), + ); + await expect( + sessions[128]!.complete({ + position: 14, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + status: "ready", + value: { + issues: [{ reason: "catalog-overloaded" }], + }, + }); + service.dispose(); + }); + + it("accepts omitted and explicit-undefined completion budgets", () => { + const first = createSqlLanguageService({ + completion: {}, + dialects: [duckdb], + }); + const second = createSqlLanguageService({ + completion: { catalogResponseBudgetMs: undefined }, + dialects: [duckdb], + }); + first.dispose(); + second.dispose(); + }); + + it("contains synchronous invalidation while replacing a refresh intent", async () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + let searchCount = 0; + const service = createSqlLanguageService({ + catalog: { + id: "reentrant-invalidation", + search: async () => { + searchCount += 1; + if (searchCount === 1) { + return new Promise(() => { + // Retained as the first completion's refresh intent. + }); + } + invalidate?.({ + epoch: { generation: 1, token: "synchronous" }, + }); + return { + coverage: { kind: "complete" }, + epoch: { generation: 1, token: "synchronous" }, + relations: [], + status: "ready", + }; + }, + subscribe: (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const text = "SELECT * FROM a; SELECT * FROM b"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:reentrant" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + await session.complete({ + position: text.indexOf("a;") + 1, + trigger: { kind: "invoked" }, + }); + await expect( + session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + reason: "superseded", + status: "cancelled", + }); + expect(searchCount).toBe(2); + service.dispose(); + }); + + it("contains a synchronous document update from provider search", async () => { + let session: + | ReturnType["openDocument"]> + | undefined; + const service = catalogService({ + id: "reentrant-update", + search: async () => { + if (session) { + session.update({ + baseRevision: session.revision, + context: { + catalog: { scope: "connection:reentrant-update" }, + dialect: "duckdb", + engine: "updated", + }, + }); + } + return { + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }; + }, + }); + session = service.openDocument({ + context: { + catalog: { scope: "connection:reentrant-update" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await expect( + session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + reason: "superseded", + status: "cancelled", + }); + service.dispose(); + }); + + it("ignores a hostile late abort callback after completion", async () => { + let searchCount = 0; + let resolveSecond: + | (( + response: Awaited< + ReturnType + >, + ) => void) + | undefined; + let secondSignal: AbortSignal | undefined; + const secondResult = new Promise< + Awaited> + >((resolve) => { + resolveSecond = resolve; + }); + const service = catalogService({ + id: "late-abort", + search: async (_request, signal) => { + searchCount += 1; + if (searchCount === 1) { + return { + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }; + } + secondSignal = signal; + return secondResult; + }, + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:late-abort" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const controller = new AbortController(); + let captured: + | EventListenerOrEventListenerObject + | undefined; + const nativeAdd = controller.signal.addEventListener.bind( + controller.signal, + ); + Object.defineProperty(controller.signal, "addEventListener", { + configurable: true, + value: ( + type: string, + listener: EventListenerOrEventListenerObject, + options?: AddEventListenerOptions, + ) => { + if (type === "abort") captured = listener; + nativeAdd(type, listener, options); + }, + }); + await session.complete({ + position: 14, + signal: controller.signal, + trigger: { kind: "invoked" }, + }); + session.update({ + baseRevision: session.revision, + document: { + kind: "replace", + text: "SELECT * FROM u", + }, + embeddedRegions: [], + }); + const current = session.complete({ + position: 15, + trigger: { kind: "invoked" }, + }); + await Promise.resolve(); + if (typeof captured === "function") { + captured(new Event("abort")); + } else { + captured?.handleEvent(new Event("abort")); + } + expect(secondSignal?.aborted).toBe(false); + resolveSecond?.({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }); + await expect(current).resolves.toMatchObject({ + status: "ready", + }); + service.dispose(); + }); + + it("normalizes a hostile range-inspection failure", () => { + const { service, session } = openSession(""); + const hostileChange = new Proxy( + { from: 0, insert: "x", to: 0 }, + { + getOwnPropertyDescriptor() { + throw new Error("hostile range"); + }, + }, + ); + expectSessionError("invalid-change", () => { + session.update({ + baseRevision: session.revision, + document: { + changes: [hostileChange], + kind: "changes", + }, + embeddedRegions: [], + }); + }); + service.dispose(); + }); + + it("accepts a host timer whose opaque handle is null", async () => { + const nativeSetTimeout = globalThis.setTimeout; + const nativeClearTimeout = globalThis.clearTimeout; + let scheduled = 0; + const cleared: unknown[] = []; + Object.defineProperty(globalThis, "setTimeout", { + configurable: true, + value: () => { + scheduled += 1; + return null; + }, + writable: true, + }); + Object.defineProperty(globalThis, "clearTimeout", { + configurable: true, + value: (handle: unknown) => { + cleared.push(handle); + scheduled -= 1; + }, + writable: true, + }); + try { + const service = catalogService(); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:null-timer" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await expect( + session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ status: "ready" }); + expect(cleared).toContain(null); + service.dispose(); + expect(scheduled).toBe(0); + + const loadingService = catalogService({ + id: "null-loading-timer", + search: async () => ({ + epoch: { generation: 0, token: "loading" }, + status: "loading", + }), + }); + const loadingSession = loadingService.openDocument({ + context: { + catalog: { scope: "connection:null-loading-timer" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await expect(loadingSession.complete({ + position: 14, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + status: "ready", + value: { + issues: [{ reason: "catalog-loading" }], + }, + }); + const clearedBeforeDispose = cleared.length; + loadingService.dispose(); + expect(cleared.length).toBeGreaterThan(clearedBeforeDispose); + expect(cleared.at(-1)).toBe(null); + expect(scheduled).toBe(0); + } finally { + Object.defineProperty(globalThis, "setTimeout", { + configurable: true, + value: nativeSetTimeout, + writable: true, + }); + Object.defineProperty(globalThis, "clearTimeout", { + configurable: true, + value: nativeClearTimeout, + writable: true, + }); + } + }); + + it("handles synchronous terminal-intent timer completion", async () => { + const nativeSetTimeout = globalThis.setTimeout; + const nativeClearTimeout = globalThis.clearTimeout; + const cleared: unknown[] = []; + let synchronousCalls = 0; + Object.defineProperty(globalThis, "setTimeout", { + configurable: true, + value: ( + callback: TimerHandler, + delay?: number, + ...arguments_: unknown[] + ) => { + if (delay === 1_000) { + synchronousCalls += 1; + if (typeof callback === "function") { + Reflect.apply(callback, undefined, arguments_); + } + return null; + } + return nativeSetTimeout(callback, delay, ...arguments_); + }, + writable: true, + }); + Object.defineProperty(globalThis, "clearTimeout", { + configurable: true, + value: (handle: ReturnType) => { + cleared.push(handle); + nativeClearTimeout(handle); + }, + writable: true, + }); + try { + const service = catalogService({ + id: "synchronous-intent-timer", + search: async () => ({ + epoch: { generation: 0, token: "loading" }, + status: "loading", + }), + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:synchronous-intent-timer" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await expect(session.complete({ + position: 14, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + status: "ready", + value: { + issues: [{ reason: "catalog-loading" }], + }, + }); + expect(synchronousCalls).toBe(1); + expect(cleared).toContain(null); + service.dispose(); + } finally { + Object.defineProperty(globalThis, "setTimeout", { + configurable: true, + value: nativeSetTimeout, + writable: true, + }); + Object.defineProperty(globalThis, "clearTimeout", { + configurable: true, + value: nativeClearTimeout, + writable: true, + }); + } + }); + + it("does not let a stale intent callback erase a newer timer", async () => { + const nativeSetTimeout = globalThis.setTimeout; + const nativeClearTimeout = globalThis.clearTimeout; + const callbacks: Array<() => void> = []; + const handles: object[] = []; + const cleared: number[] = []; + Object.defineProperty(globalThis, "setTimeout", { + configurable: true, + value: ( + callback: TimerHandler, + delay?: number, + ...arguments_: unknown[] + ) => { + if (delay !== 1_000) { + return nativeSetTimeout(callback, delay, ...arguments_); + } + if (typeof callback !== "function") { + throw new Error("Intent timer callback must be a function"); + } + callbacks.push(() => { + Reflect.apply(callback, undefined, arguments_); + }); + const handle = Object.freeze({ id: handles.length }); + handles.push(handle); + return handle; + }, + writable: true, + }); + Object.defineProperty(globalThis, "clearTimeout", { + configurable: true, + value: (handle: unknown) => { + const index = handles.findIndex( + (candidate) => candidate === handle, + ); + if (index >= 0) { + cleared.push(index); + } else { + Reflect.apply(nativeClearTimeout, globalThis, [handle]); + } + }, + writable: true, + }); + try { + const service = catalogService({ + id: "stale-intent-timer", + search: async () => ({ + epoch: { generation: 0, token: "loading" }, + status: "loading", + }), + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:stale-intent-timer" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + await session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + expect(callbacks).toHaveLength(2); + expect(cleared).toContain(0); + + callbacks[0]?.(); + session.dispose(); + + expect(cleared).toContain(1); + service.dispose(); + } finally { + Object.defineProperty(globalThis, "setTimeout", { + configurable: true, + value: nativeSetTimeout, + writable: true, + }); + Object.defineProperty(globalThis, "clearTimeout", { + configurable: true, + value: nativeClearTimeout, + writable: true, + }); + } + }); + + it("consumes settlement racing the soft response timer", async () => { + let resolveSearch: + | (( + response: Awaited< + ReturnType + >, + ) => void) + | undefined; + const searchResult = new Promise< + Awaited> + >((resolve) => { + resolveSearch = resolve; + }); + const nativeSetTimeout = globalThis.setTimeout; + let injected = false; + Object.defineProperty(globalThis, "setTimeout", { + configurable: true, + value: ( + callback: TimerHandler, + delay?: number, + ...arguments_: unknown[] + ) => + nativeSetTimeout( + (...timerArguments: unknown[]) => { + if (delay === 0 && !injected) { + injected = true; + resolveSearch?.({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }); + } + if (typeof callback === "function") { + Reflect.apply(callback, undefined, timerArguments); + } + }, + delay, + ...arguments_, + ), + writable: true, + }); + try { + const service = createSqlLanguageService({ + catalog: { + id: "settlement-race", + search: async () => searchResult, + }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:settlement-race" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const events: string[] = []; + session.onDidChange((event) => { + events.push(event.reason); + }); + await expect(session.complete({ + position: 14, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + sources: [ + { + coverage: "complete", + outcome: "ready", + providerId: "settlement-race", + }, + ], + status: "ready", + value: { + isIncomplete: false, + issues: [], + }, + }); + expect(events).toEqual([]); + service.dispose(); + } finally { + Object.defineProperty(globalThis, "setTimeout", { + configurable: true, + value: nativeSetTimeout, + writable: true, + }); + } + }); + + it("fails catalog ownership closed after per-scope membership capacity", async () => { + const service = catalogService(); + const sessions = Array.from({ length: 257 }, () => + service.openDocument({ + context: { + catalog: { scope: "connection:membership-capacity" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }), + ); + await expect( + sessions[256]!.complete({ + position: 14, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + status: "ready", + value: { + issues: [{ reason: "catalog-overloaded" }], + }, + }); + service.dispose(); + }); + + it("forwards a validated ordered catalog search path", async () => { + let observed: + | Parameters[0] + | undefined; + const service = catalogService({ + id: "search-path", + search: async (request) => { + observed = request; + return { + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }; + }, + }); + const searchPath = [ + [ + { quoted: false, value: "memory" }, + { quoted: true, value: "Main Schema" }, + ], + ] as const; + const session = service.openDocument({ + context: { + catalog: { + scope: "connection:search-path", + searchPath, + }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + expect(observed?.searchPaths).toEqual(searchPath); + expect(observed?.searchPaths).not.toBe(searchPath); + service.dispose(); + }); + + it("contains reentrant completion during signal registration", async () => { + const service = createSqlLanguageService({ + catalog: { + id: "reentrant-signal", + search: async () => + new Promise(() => { + // Settled by session disposal. + }), + }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:reentrant-signal" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + await session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + const controller = new AbortController(); + let nested: + | ReturnType + | undefined; + const nativeAdd = controller.signal.addEventListener.bind( + controller.signal, + ); + Object.defineProperty(controller.signal, "addEventListener", { + configurable: true, + value: ( + type: string, + listener: EventListenerOrEventListenerObject, + options?: AddEventListenerOptions, + ) => { + if (!nested) { + nested = session.complete({ + position: 14, + trigger: { kind: "invoked" }, + }); + } + nativeAdd(type, listener, options); + }, + }); + await expect( + session.complete({ + position: 0, + signal: controller.signal, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + reason: "inactive", + status: "unavailable", + }); + service.dispose(); + await expect(nested).resolves.toMatchObject({ + reason: "disposed", + status: "cancelled", + }); + }); + + it("stops catalog event delivery when a listener disposes the session", () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = catalogService({ + id: "dispose-listener", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }), + subscribe: (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:dispose-listener" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const events: string[] = []; + session.onDidChange((event) => { + events.push(event.reason); + session.dispose(); + }); + session.onDidChange((event) => { + events.push(`${event.reason}:late`); + }); + + invalidate?.({ + epoch: { generation: 1, token: "changed" }, + }); + + expect(events).toEqual(["catalog"]); + service.dispose(); + }); +}); diff --git a/src/vnext/codemirror/relation-completion-types.ts b/src/vnext/codemirror/relation-completion-types.ts index 8c736d7..53aac2f 100644 --- a/src/vnext/codemirror/relation-completion-types.ts +++ b/src/vnext/codemirror/relation-completion-types.ts @@ -1,4 +1,4 @@ -import type { SqlRelationCompletionItem } from "../relation-completion-types.js"; +import type { SqlCompletionItem } from "../relation-completion-types.js"; // Provisional CodeMirror-only boundary; the framework-independent core stays DOM-free. export interface SqlCompletionInfoResolverContext { @@ -11,7 +11,7 @@ export interface SqlDisposableCompletionInfo { } export type SqlCompletionInfoResolver = ( - item: SqlRelationCompletionItem, + item: SqlCompletionItem, context: SqlCompletionInfoResolverContext, ) => | SqlDisposableCompletionInfo diff --git a/src/vnext/index.ts b/src/vnext/index.ts index d850c9f..8204d7b 100644 --- a/src/vnext/index.ts +++ b/src/vnext/index.ts @@ -28,3 +28,26 @@ export type { SqlTextRange, } from "./types.js"; export { SqlSessionError } from "./types.js"; +export type { + SqlCanonicalRelationPath, + SqlCatalogEpoch, + SqlCatalogFailureCode, + SqlCatalogProviderReport, + SqlCatalogReadyCoverage, + SqlCatalogRelation, + SqlCatalogRelationKind, + SqlCatalogRetryPolicy, + SqlCatalogSearchRequest, + SqlCatalogSearchResponse, + SqlCompletionCancellationReason, + SqlCompletionIssue, + SqlCompletionRequest, + SqlCompletionTrigger, + SqlDisposable, + SqlRelationCatalogProvider, + SqlCompletionItem, + SqlCompletionList, + SqlCompletionResult, + SqlSessionChangeEvent, + SqlSessionChangeReason, +} from "./relation-completion-types.js"; diff --git a/src/vnext/relation-completion-types.ts b/src/vnext/relation-completion-types.ts index 1222d10..56c8952 100644 --- a/src/vnext/relation-completion-types.ts +++ b/src/vnext/relation-completion-types.ts @@ -1,6 +1,4 @@ import type { - SqlDocumentContext, - SqlDocumentUpdate, SqlIdentifierComponent, SqlIdentifierPath, SqlRevision, @@ -196,6 +194,7 @@ export interface SqlSessionChangeEvent { export type SqlCompletionTrigger = | { + readonly character?: never; readonly kind: "invoked"; } | { @@ -206,7 +205,7 @@ export type SqlCompletionTrigger = export interface SqlCompletionRequest { readonly position: number; readonly trigger: SqlCompletionTrigger; - readonly signal?: AbortSignal; + readonly signal?: AbortSignal | undefined; } export interface SqlCteCompletionProvenance { @@ -226,12 +225,14 @@ interface SqlCompletionItemBase { readonly detail?: string; } -export type SqlRelationCompletionItem = +export type SqlCompletionItem = | (SqlCompletionItemBase & { + readonly kind: "relation"; readonly relationKind: "cte"; readonly provenance: SqlCteCompletionProvenance; }) | (SqlCompletionItemBase & { + readonly kind: "relation"; readonly relationKind: SqlCatalogRelationKind; readonly provenance: SqlCatalogCompletionProvenance; }); @@ -257,14 +258,14 @@ export type SqlCompletionIssue = | "result-limit"; }; -export type SqlRelationCompletionList = +export type SqlCompletionList = | { - readonly items: readonly SqlRelationCompletionItem[]; + readonly items: readonly SqlCompletionItem[]; readonly isIncomplete: false; readonly issues: readonly []; } | { - readonly items: readonly SqlRelationCompletionItem[]; + readonly items: readonly SqlCompletionItem[]; readonly isIncomplete: true; readonly issues: readonly [ SqlCompletionIssue, @@ -288,7 +289,6 @@ export type SqlCatalogProviderUnavailableReason = | "queue-overloaded" | "queue-timeout" | "execution-timeout" - | "synchronous-timeout" | "provider-rejected" | "malformed-response"; @@ -320,11 +320,11 @@ export interface SqlServiceFailure { readonly retryable: boolean; } -export type SqlRelationCompletionResult = +export type SqlCompletionResult = | { readonly status: "ready"; readonly revision: SqlRevision; - readonly value: SqlRelationCompletionList; + readonly value: SqlCompletionList; readonly sources: readonly SqlCatalogProviderReport[]; } | { @@ -343,20 +343,3 @@ export type SqlRelationCompletionResult = readonly revision: SqlRevision; readonly failure: SqlServiceFailure; }; - -export interface SqlRelationCompletionSession< - Context extends SqlDocumentContext, -> { - readonly revision: SqlRevision; - readonly update: ( - transaction: SqlDocumentUpdate, - ) => SqlRevision; - readonly complete: ( - request: SqlCompletionRequest, - ) => Promise; - readonly onDidChange: ( - listener: (event: SqlSessionChangeEvent) => void, - ) => SqlDisposable; - readonly isCurrent: (revision: SqlRevision) => boolean; - readonly dispose: () => void; -} diff --git a/src/vnext/relation-completion.ts b/src/vnext/relation-completion.ts new file mode 100644 index 0000000..a0d2668 --- /dev/null +++ b/src/vnext/relation-completion.ts @@ -0,0 +1,522 @@ +import type { + SqlCatalogProviderReport, + SqlCompletionIssue, + SqlCompletionItem, + SqlCompletionList, +} from "./relation-completion-types.js"; +import type { + SqlCatalogSearchWorkOutcome, +} from "./relation-catalog-search-work.js"; +import type { + SqlRelationDialectRuntime, +} from "./relation-dialect.js"; +import type { + SqlLocalRelationSiteResult, +} from "./local-relation-site.js"; +import type { + SqlIdentifierComponent, + SqlTextRange, +} from "./types.js"; + +export const MAX_RELATION_COMPLETION_RESULTS = 100; + +type SqlReadyLocalRelationSite = Extract< + SqlLocalRelationSiteResult, + { readonly status: "ready" } +>; + +type SqlComposableCatalogUnavailableReason = + | "execution-timeout" + | "malformed-response" + | "overloaded" + | "provider-failed" + | "queue-timeout"; + +export type SqlComposableCatalogOutcome = + | Extract< + SqlCatalogSearchWorkOutcome, + { readonly status: "usable" } + > + | { + readonly status: "loading"; + } + | { + readonly status: "unavailable"; + readonly reason: SqlComposableCatalogUnavailableReason; + } + | null; + +export interface SqlRelationCompletionCompositionInput { + readonly catalogOutcome: SqlComposableCatalogOutcome; + readonly dialect: SqlRelationDialectRuntime; + readonly localSite: SqlReadyLocalRelationSite; + readonly providerId: string | null; + readonly remainingIntentLeaseMs: number; + readonly replacementRange: SqlTextRange; + readonly statementOffset: number; +} + +export interface SqlRelationCompletionComposition { + readonly sources: readonly SqlCatalogProviderReport[]; + readonly value: SqlCompletionList; +} + +interface RankedCteItem { + readonly item: Extract< + SqlCompletionItem, + { readonly relationKind: "cte" } + >; + readonly path: string; +} + +interface RankedCatalogItem { + readonly completionPathLength: number; + readonly item: Exclude< + SqlCompletionItem, + { readonly relationKind: "cte" } + >; + readonly label: string; + readonly matchQuality: "exact" | "equivalent"; + readonly path: string; +} + +const CATALOG_KIND_ORDER = Object.freeze({ + "temporary-table": 0, + table: 1, + view: 2, + "materialized-view": 3, + "external-relation": 4, +} as const); + +const ISSUE_ORDER = Object.freeze([ + "query-site-recovery", + "opaque-template-context", + "cte-scope-uncertainty", + "recursive-cte-uncertainty", + "catalog-loading", + "catalog-partial", + "catalog-paginated", + "catalog-failed", + "catalog-malformed", + "catalog-overloaded", + "catalog-queue-timeout", + "catalog-timeout", + "result-limit", +] as const); + +type OrderedIssueReason = (typeof ISSUE_ORDER)[number]; + +function compareCodeUnits(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function compareCteItems( + left: RankedCteItem, + right: RankedCteItem, +): number { + return ( + compareCodeUnits(left.item.label, right.item.label) || + compareCodeUnits(left.path, right.path) || + left.item.provenance.declarationPosition - + right.item.provenance.declarationPosition + ); +} + +function compareCatalogItems( + left: RankedCatalogItem, + right: RankedCatalogItem, +): number { + return ( + (left.matchQuality === right.matchQuality + ? 0 + : left.matchQuality === "exact" + ? -1 + : 1) || + left.completionPathLength - right.completionPathLength || + CATALOG_KIND_ORDER[left.item.relationKind] - + CATALOG_KIND_ORDER[right.item.relationKind] || + compareCodeUnits(left.label, right.label) || + compareCodeUnits(left.path, right.path) || + compareCodeUnits( + left.item.provenance.entityId, + right.item.provenance.entityId, + ) + ); +} + +function addLocalIssues( + input: SqlRelationCompletionCompositionInput, + issues: Set, +): void { + const { localSite } = input; + if (localSite.querySite.recognition.quality === "recovered") { + issues.add("query-site-recovery"); + } + if ( + localSite.querySite.recognition.issues.some( + (issue) => issue === "opaque-template-context", + ) + ) { + issues.add("opaque-template-context"); + } + if (localSite.local.kind !== "unqualified") return; + const visibility = localSite.local.cteVisibility; + if ( + visibility.quality === "recovered" || + visibility.shadowing.coverage === "unknown" || + visibility.issues.length > 0 + ) { + issues.add("cte-scope-uncertainty"); + } + if ( + visibility.issues.includes("opaque-template-context") + ) { + issues.add("opaque-template-context"); + } + if ( + visibility.issues.includes("recursive-cte-position") + ) { + issues.add("recursive-cte-uncertainty"); + } +} + +function createCteItems( + input: SqlRelationCompletionCompositionInput, + issues: Set, +): RankedCteItem[] { + if (input.localSite.local.kind !== "unqualified") { + return []; + } + const items: RankedCteItem[] = []; + const prefix = input.localSite.querySite.prefix; + for (const cte of input.localSite.local.cteVisibility.ctes) { + let match: ReturnType< + SqlRelationDialectRuntime["completion"]["cteIdentifierMatchesPrefix"] + >; + try { + match = + input.dialect.completion.cteIdentifierMatchesPrefix( + cte.name, + prefix, + ); + } catch { + match = "unknown"; + } + if (match === "unknown") { + issues.add("cte-scope-uncertainty"); + continue; + } + if (match === "no-match") continue; + const declarationPosition = + input.statementOffset + cte.declarationPosition; + const item = Object.freeze({ + edit: Object.freeze({ + from: input.replacementRange.from, + insert: cte.sourceSpelling, + to: input.replacementRange.to, + }), + label: cte.name.value, + kind: "relation" as const, + provenance: Object.freeze({ + declarationPosition, + kind: "cte" as const, + }), + relationKind: "cte" as const, + }); + items.push({ item, path: cte.sourceSpelling }); + } + items.sort(compareCteItems); + return items; +} + +function cteShadowsCatalogRelation( + input: SqlRelationCompletionCompositionInput, + relationName: SqlIdentifierComponent, + issues: Set, +): boolean { + if (input.localSite.local.kind !== "unqualified") { + return false; + } + const shadowing = + input.localSite.local.cteVisibility.shadowing; + if (shadowing.coverage === "unknown") { + issues.add("cte-scope-uncertainty"); + return false; + } + for (const name of shadowing.names) { + let comparison: ReturnType< + SqlRelationDialectRuntime["completion"]["compareCteIdentifiers"] + >; + try { + comparison = + input.dialect.completion.compareCteIdentifiers( + name, + relationName, + ); + } catch { + comparison = "unknown"; + } + if (comparison === "equal") return true; + if (comparison === "unknown") { + issues.add("cte-scope-uncertainty"); + } + } + return false; +} + +function createCatalogItems( + input: SqlRelationCompletionCompositionInput, + issues: Set, +): RankedCatalogItem[] { + const outcome = input.catalogOutcome; + if ( + outcome === null || + outcome.status !== "usable" || + outcome.response.status !== "ready" || + input.providerId === null + ) { + return []; + } + const items: RankedCatalogItem[] = []; + for (const relation of outcome.response.relations) { + const relationName = + relation.canonicalPath[ + relation.canonicalPath.length - 1 + ]; + if (!relationName || relationName.role !== "relation") continue; + if ( + relation.completionPath.length === 1 && + cteShadowsCatalogRelation( + input, + relationName, + issues, + ) + ) { + continue; + } + const base = { + edit: Object.freeze({ + from: input.replacementRange.from, + insert: relation.completionText, + to: input.replacementRange.to, + }), + label: relationName.value, + kind: "relation" as const, + provenance: Object.freeze({ + entityId: relation.entityId, + kind: "catalog" as const, + providerId: input.providerId, + }), + relationKind: relation.relationKind, + }; + const item = Object.freeze( + relation.detail === undefined + ? base + : { ...base, detail: relation.detail }, + ); + const renderedLabel = + input.dialect.completion.renderRelationPath( + Object.freeze([relationName]), + ); + items.push({ + completionPathLength: relation.completionPath.length, + item, + label: + renderedLabel.status === "rendered" + ? renderedLabel.text + : relationName.value, + matchQuality: relation.matchQuality, + path: relation.completionText, + }); + } + items.sort(compareCatalogItems); + return items; +} + +function unavailableIssue( + reason: SqlComposableCatalogUnavailableReason, +): OrderedIssueReason { + switch (reason) { + case "execution-timeout": + return "catalog-timeout"; + case "malformed-response": + return "catalog-malformed"; + case "overloaded": + return "catalog-overloaded"; + case "provider-failed": + return "catalog-failed"; + case "queue-timeout": + return "catalog-queue-timeout"; + } +} + +function unavailableReportReason( + reason: SqlComposableCatalogUnavailableReason, +): Extract< + SqlCatalogProviderReport, + { readonly outcome: "unavailable" } +>["reason"] { + switch (reason) { + case "execution-timeout": + return "execution-timeout"; + case "malformed-response": + return "malformed-response"; + case "overloaded": + return "queue-overloaded"; + case "provider-failed": + return "provider-rejected"; + case "queue-timeout": + return "queue-timeout"; + } +} + +function addCatalogEvidence( + input: SqlRelationCompletionCompositionInput, + issues: Set, +): readonly SqlCatalogProviderReport[] { + const outcome = input.catalogOutcome; + const providerId = input.providerId; + if (outcome === null || providerId === null) return Object.freeze([]); + if (outcome.status === "loading") { + issues.add("catalog-loading"); + return Object.freeze([ + Object.freeze({ + feature: "relation-catalog" as const, + outcome: "loading" as const, + providerId, + }), + ]); + } + if (outcome.status === "unavailable") { + issues.add(unavailableIssue(outcome.reason)); + return Object.freeze([ + Object.freeze({ + feature: "relation-catalog" as const, + outcome: "unavailable" as const, + providerId, + reason: unavailableReportReason(outcome.reason), + }), + ]); + } + const response = outcome.response; + if (response.status === "loading") { + issues.add("catalog-loading"); + return Object.freeze([ + Object.freeze({ + feature: "relation-catalog" as const, + outcome: "loading" as const, + providerId, + }), + ]); + } + if (response.status === "failed") { + issues.add("catalog-failed"); + return Object.freeze([ + Object.freeze({ + code: response.code, + feature: "relation-catalog" as const, + outcome: "failed" as const, + providerId, + retry: response.retry, + }), + ]); + } + if (response.coverage.kind === "partial") { + issues.add("catalog-partial"); + } else if (response.coverage.kind === "paginated") { + issues.add("catalog-paginated"); + } + return Object.freeze([ + Object.freeze({ + coverage: response.coverage.kind, + feature: "relation-catalog" as const, + outcome: "ready" as const, + providerId, + }), + ]); +} + +function freezeIssues( + reasons: ReadonlySet, + remainingIntentLeaseMs: number, +): readonly SqlCompletionIssue[] { + const issues: SqlCompletionIssue[] = []; + for (const reason of ISSUE_ORDER) { + if (!reasons.has(reason)) continue; + issues.push( + reason === "catalog-loading" + ? Object.freeze({ + reason, + remainingIntentLeaseMs, + }) + : Object.freeze({ reason }), + ); + } + return Object.freeze(issues); +} + +function completeList( + items: readonly SqlCompletionItem[], +): Extract< + SqlCompletionList, + { readonly isIncomplete: false } +> { + const noIssues: readonly [] = Object.freeze([]); + return Object.freeze({ + isIncomplete: false, + issues: noIssues, + items, + }); +} + +function incompleteList( + items: readonly SqlCompletionItem[], + firstIssue: SqlCompletionIssue, + remainingIssues: readonly SqlCompletionIssue[], +): Extract< + SqlCompletionList, + { readonly isIncomplete: true } +> { + const issues: readonly [ + SqlCompletionIssue, + ...SqlCompletionIssue[], + ] = Object.freeze([firstIssue, ...remainingIssues]); + return Object.freeze({ + isIncomplete: true, + issues, + items, + }); +} + +export function composeSqlRelationCompletion( + input: SqlRelationCompletionCompositionInput, +): SqlRelationCompletionComposition { + const issues = new Set(); + addLocalIssues(input, issues); + const ctes = createCteItems(input, issues); + const catalog = createCatalogItems(input, issues); + const ranked = [ + ...ctes.map(({ item }) => item), + ...catalog.map(({ item }) => item), + ]; + if (ranked.length > MAX_RELATION_COMPLETION_RESULTS) { + ranked.length = MAX_RELATION_COMPLETION_RESULTS; + issues.add("result-limit"); + } + const sources = addCatalogEvidence(input, issues); + const frozenItems = Object.freeze(ranked); + const frozenIssues = freezeIssues( + issues, + input.remainingIntentLeaseMs, + ); + const firstIssue = frozenIssues[0]; + const value = + firstIssue === undefined + ? completeList(frozenItems) + : incompleteList( + frozenItems, + firstIssue, + frozenIssues.slice(1), + ); + return Object.freeze({ sources, value }); +} diff --git a/src/vnext/session.ts b/src/vnext/session.ts index 7306d16..80b1169 100644 --- a/src/vnext/session.ts +++ b/src/vnext/session.ts @@ -6,15 +6,50 @@ import type { SqlLanguageService, SqlLanguageServiceOptions, SqlRevision, + SqlIdentifierPath, SqlTextChange, SqlTextRange, } from "./types.js"; import { buildSqlStatementIndex, + findSqlStatementSlot, type SqlLexicalProfile, type SqlStatementIndex, + type SqlStatementSlot, updateSqlStatementIndex, } from "./statement-index.js"; +import { + captureSqlRelationCatalogProvider, + MAX_CATALOG_IDENTIFIER_LENGTH, + MAX_CATALOG_SCOPE_LENGTH, + MAX_CATALOG_SEARCH_PATH_COMPONENTS, + MAX_CATALOG_SEARCH_PATHS, +} from "./relation-catalog-boundary.js"; +import { + createSqlCatalogSearchWorkCoordinator, + type SqlCatalogSearchWorkCoordinator, + type SqlCatalogSearchWorkOwner, + type SqlCatalogSearchWorkOutcome, + type SqlCatalogSearchWorkTicket, +} from "./relation-catalog-search-work.js"; +import { + analyzeSqlLocalRelationSite, + prepareSqlLocalRelationStatement, + type SqlLocalRelationStatementPreparation, + type SqlLocalRelationSiteResult, +} from "./local-relation-site.js"; +import { + composeSqlRelationCompletion, + MAX_RELATION_COMPLETION_RESULTS, + type SqlComposableCatalogOutcome, +} from "./relation-completion.js"; +import type { + SqlCompletionRequest, + SqlDisposable, + SqlCompletionResult, + SqlSessionChangeEvent, + SqlSessionChangeReason, +} from "./relation-completion-types.js"; import { BIGQUERY_SQL_RELATION_DIALECT, DREMIO_SQL_RELATION_DIALECT, @@ -45,6 +80,35 @@ const MAX_CONTEXT_STRING_LENGTH = 1_000_000; const MAX_CONTEXT_ARRAY_LENGTH = 50_000; const MAX_CHANGES_PER_UPDATE = 10_000; const MAX_DIALECTS = 1_000; +const DEFAULT_CATALOG_RESPONSE_BUDGET_MS = 40; +const MAX_CATALOG_RESPONSE_BUDGET_MS = 50; +const TERMINAL_LOADING_INTENT_LEASE_MS = 1_000; + +interface ResolvedCatalogContext { + readonly scope: string; + readonly searchPaths: readonly SqlIdentifierPath[]; +} + +interface CompletionRequestState { + cancelReason: "caller" | "disposed" | "superseded" | null; + readonly revision: SqlRevision; + ticket: SqlCatalogSearchWorkTicket | null; + readonly token: object; +} + +interface SessionChangeSubscription { + active: boolean; + readonly listener: (event: SqlSessionChangeEvent) => void; +} + +interface SessionTimerCell { + active: boolean; + handle: ReturnType | undefined; +} + +interface CompletionConfiguration { + readonly catalogResponseBudgetMs: number; +} interface SqlDialectRuntime { readonly dialect: SqlDialect; @@ -352,6 +416,89 @@ function resolveDialectRuntime( return runtime; } +function resolveCatalogContext( + context: SqlDocumentContext, +): ResolvedCatalogContext | null { + const catalog = context.catalog; + if (!catalog) return null; + if ( + typeof catalog !== "object" || + typeof catalog.scope !== "string" || + catalog.scope.length === 0 || + catalog.scope.length > MAX_CATALOG_SCOPE_LENGTH + ) { + throw new SqlSessionError( + "invalid-context", + "SQL catalog context has an invalid scope", + ); + } + const candidatePaths = catalog.searchPath ?? []; + if ( + !Array.isArray(candidatePaths) || + candidatePaths.length > MAX_CATALOG_SEARCH_PATHS + ) { + throw new SqlSessionError( + "invalid-context", + "SQL catalog search paths are invalid", + ); + } + for (const path of candidatePaths) { + if ( + !Array.isArray(path) || + path.length === 0 || + path.length > MAX_CATALOG_SEARCH_PATH_COMPONENTS + ) { + throw new SqlSessionError( + "invalid-context", + "SQL catalog search path is invalid", + ); + } + for (const component of path) { + if ( + !component || + typeof component !== "object" || + typeof component.value !== "string" || + component.value.length === 0 || + component.value.length > MAX_CATALOG_IDENTIFIER_LENGTH || + typeof component.quoted !== "boolean" + ) { + throw new SqlSessionError( + "invalid-context", + "SQL catalog search path component is invalid", + ); + } + } + } + return Object.freeze({ + scope: catalog.scope, + searchPaths: candidatePaths, + }); +} + +function unavailableCompletionReason( + local: Exclude, +): "inactive" | "unsupported-query-site" | "opaque-statement" | + "ambiguous-query-site" | "resource-limit" { + if (local.status === "inactive") return "inactive"; + switch (local.reason) { + case "opaque-statement": + return "opaque-statement"; + case "resource-limit": + return "resource-limit"; + case "unsupported-query-site": + return "unsupported-query-site"; + case "ambiguous-query-site": + return "ambiguous-query-site"; + } +} + +function completionCancellation( + revision: SqlRevision, + reason: "caller" | "disposed" | "superseded", +): SqlCompletionResult { + return Object.freeze({ reason, revision, status: "cancelled" }); +} + interface MissingDataProperty { readonly found: false; } @@ -588,22 +735,47 @@ interface StatementIndexCache { readonly sourceSequence: number; } +interface LocalRelationStatementCache { + readonly dialect: SqlRelationDialectRuntime; + readonly index: SqlStatementIndex; + readonly preparation: SqlLocalRelationStatementPreparation; + readonly slot: SqlStatementSlot; + readonly sourceSequence: number; +} + export class DefaultSqlDocumentSession implements SqlDocumentSession { + readonly #catalogCoordinator: SqlCatalogSearchWorkCoordinator | null; + readonly #catalogResponseBudgetMs: number; readonly #dialects: ReadonlyMap; readonly #onDispose: () => void; + readonly #listeners = new Set(); + #activeCompletion: CompletionRequestState | null = null; + #catalogOwner: SqlCatalogSearchWorkOwner | null = null; + #catalogOwnerDialect: SqlRelationDialectRuntime | null = null; + #catalogOwnerScope: string | null = null; #disposed = false; + #localRelationStatementCache: + | LocalRelationStatementCache + | null = null; + #refreshIntent: CompletionRequestState | null = null; #snapshot: SessionSnapshot; #statementIndexCache: StatementIndexCache | null = null; + #terminalIntentTimer: SessionTimerCell | null = null; #updating = false; constructor( source: SqlSourceSnapshot, context: Context, dialects: ReadonlyMap, + catalogCoordinator: SqlCatalogSearchWorkCoordinator | null, + completion: CompletionConfiguration, onDispose: () => void, ) { + this.#catalogCoordinator = catalogCoordinator; + this.#catalogResponseBudgetMs = + completion.catalogResponseBudgetMs; this.#dialects = dialects; this.#onDispose = onDispose; const sequence = 0; @@ -621,6 +793,7 @@ export class DefaultSqlDocumentSession source, sourceSequence, }); + this.#replaceCatalogOwner(); } get revision(): SqlRevision { @@ -662,6 +835,569 @@ export class DefaultSqlDocumentSession return index; } + #clearTerminalIntent(): void { + const cell = this.#terminalIntentTimer; + if (!cell) return; + this.#terminalIntentTimer = null; + cell.active = false; + clearTimeout(cell.handle); + } + + #dispatchChange( + revision: SqlRevision, + reason: SqlSessionChangeReason, + ): void { + if (this.#disposed || revision !== this.#snapshot.revision) { + return; + } + const event = Object.freeze({ reason, revision }); + for (const subscription of Array.from(this.#listeners)) { + if ( + this.#disposed || + revision !== this.#snapshot.revision + ) { + return; + } + if ( + !subscription.active || + !this.#listeners.has(subscription) + ) { + continue; + } + try { + Reflect.apply(subscription.listener, undefined, [event]); + } catch { + // Listener failures do not interrupt coordinator state changes. + } + } + } + + #prepareServiceChange( + reason: SqlSessionChangeReason, + expected?: CompletionRequestState, + ): (() => undefined) | null { + if ( + this.#disposed || + (expected !== undefined && + (this.#activeCompletion !== expected && + this.#refreshIntent !== expected || + expected.cancelReason !== null || + expected.revision !== this.#snapshot.revision)) + ) { + return null; + } + const previous = this.#snapshot; + const revision = createSqlRevisionToken(); + this.#snapshot = Object.freeze({ + ...previous, + revision, + sequence: previous.sequence + 1, + }); + const active = this.#activeCompletion; + const intent = this.#refreshIntent; + this.#activeCompletion = null; + this.#refreshIntent = null; + this.#clearTerminalIntent(); + if (active && active.cancelReason === null) { + active.cancelReason = "superseded"; + } + if (intent && intent !== active) { + if (intent.cancelReason === null) { + intent.cancelReason = "superseded"; + } + } + return (): undefined => { + active?.ticket?.cancel(); + if (intent !== active) intent?.ticket?.cancel(); + this.#dispatchChange(revision, reason); + return undefined; + }; + } + + #replaceCatalogOwner(): void { + const catalog = resolveCatalogContext(this.#snapshot.context); + const dialect = this.#snapshot.dialect.relationDialect; + if ( + this.#catalogOwner && + catalog && + this.#catalogOwnerScope === catalog.scope && + this.#catalogOwnerDialect === dialect + ) { + return; + } + this.#catalogOwner?.dispose(); + this.#catalogOwner = null; + this.#catalogOwnerDialect = null; + this.#catalogOwnerScope = null; + if (!catalog || !this.#catalogCoordinator || this.#disposed) { + return; + } + const prepared = this.#catalogCoordinator.prepareOwner( + catalog.scope, + dialect, + Object.freeze({ + prepareCatalogChange: (): (() => undefined) | null => + this.#prepareServiceChange("catalog"), + }), + ); + if (prepared.status !== "prepared") return; + this.#catalogOwner = prepared.owner; + this.#catalogOwnerDialect = dialect; + this.#catalogOwnerScope = catalog.scope; + const activation = prepared.owner.activate(); + if (activation.status !== "active") { + prepared.owner.dispose(); + if (this.#catalogOwner === prepared.owner) { + this.#catalogOwner = null; + this.#catalogOwnerDialect = null; + this.#catalogOwnerScope = null; + } + } + } + + readonly onDidChange = ( + listener: (event: SqlSessionChangeEvent) => void, + ): SqlDisposable => { + if (this.#disposed) { + throw new SqlSessionError( + "session-disposed", + "SQL document session is disposed", + ); + } + if (typeof listener !== "function") { + throw new SqlSessionError( + "invalid-completion-request", + "SQL session change listener must be a function", + ); + } + const subscription: SessionChangeSubscription = { + active: true, + listener, + }; + this.#listeners.add(subscription); + return Object.freeze({ + dispose: (): void => { + if (!subscription.active) return; + subscription.active = false; + this.#listeners.delete(subscription); + }, + }); + }; + + readonly complete = async ( + request: SqlCompletionRequest, + ): Promise => { + const completionStartedAt = performance.now(); + if (this.#disposed) { + throw new SqlSessionError( + "session-disposed", + "SQL document session is disposed", + ); + } + let position: number; + let signal: AbortSignal | undefined; + try { + if (request === null || typeof request !== "object") { + throw new Error(); + } + position = readRequiredDataProperty( + request, + "position", + "invalid-completion-request", + "SQL completion request", + ); + const trigger = readRequiredDataProperty( + request, + "trigger", + "invalid-completion-request", + "SQL completion request", + ); + if ( + !Number.isSafeInteger(position) || + position < 0 || + position > this.#snapshot.source.originalText.length || + trigger === null || + typeof trigger !== "object" + ) { + throw new Error(); + } + const triggerKind = readRequiredDataProperty( + trigger, + "kind", + "invalid-completion-request", + "SQL completion trigger", + ); + if ( + triggerKind !== "invoked" && + triggerKind !== "trigger-character" + ) { + throw new Error(); + } + if (triggerKind === "trigger-character") { + const character = readRequiredDataProperty( + trigger, + "character", + "invalid-completion-request", + "SQL completion trigger", + ); + if ( + typeof character !== "string" || + character.length === 0 || + character.length > 2 || + Array.from(character).length !== 1 + ) { + throw new Error(); + } + } else { + rejectOwnProperty( + trigger, + "character", + "invalid-completion-request", + "SQL completion trigger", + ); + } + const signalProperty = readOwnDataProperty( + request, + "signal", + "invalid-completion-request", + "SQL completion request", + ); + if ( + signalProperty.found && + signalProperty.value !== undefined + ) { + if (!(signalProperty.value instanceof AbortSignal)) { + throw new Error(); + } + signal = signalProperty.value; + } + } catch (error) { + if ( + error instanceof SqlSessionError && + error.code === "invalid-completion-request" + ) { + throw error; + } + throw new SqlSessionError( + "invalid-completion-request", + "SQL completion request is invalid", + ); + } + + const previousActive = this.#activeCompletion; + const previousIntent = this.#refreshIntent; + if ( + previousActive && + previousActive.cancelReason === null + ) { + previousActive.cancelReason = "superseded"; + } + if ( + previousIntent && + previousIntent !== previousActive && + previousIntent.cancelReason === null + ) { + previousIntent.cancelReason = "superseded"; + } + const cancelPrevious = (): void => { + previousActive?.ticket?.cancel(); + if (previousIntent !== previousActive) { + previousIntent?.ticket?.cancel(); + } + if (this.#refreshIntent === previousIntent) { + this.#refreshIntent = null; + } + }; + this.#clearTerminalIntent(); + const snapshot = this.#snapshot; + const active: CompletionRequestState = { + cancelReason: null, + revision: snapshot.revision, + ticket: null, + token: {}, + }; + this.#activeCompletion = active; + if (signal?.aborted) { + active.cancelReason = "caller"; + this.#activeCompletion = null; + cancelPrevious(); + return completionCancellation(snapshot.revision, "caller"); + } + const onAbort = (): void => { + if ( + this.#activeCompletion === active && + active.cancelReason === null + ) { + active.cancelReason = "caller"; + active.ticket?.cancel(); + } + }; + signal?.addEventListener("abort", onAbort, { once: true }); + + try { + const index = this.getStatementIndexForTesting(); + const slot = findSqlStatementSlot(index, position, "left"); + const cachedLocal = this.#localRelationStatementCache; + const prepared = + cachedLocal && + cachedLocal.sourceSequence === snapshot.sourceSequence && + cachedLocal.index === index && + cachedLocal.slot === slot && + cachedLocal.dialect === snapshot.dialect.relationDialect + ? cachedLocal.preparation + : prepareSqlLocalRelationStatement( + snapshot.source, + index, + slot, + snapshot.dialect.relationDialect, + ); + if (cachedLocal?.preparation !== prepared) { + this.#localRelationStatementCache = Object.freeze({ + dialect: snapshot.dialect.relationDialect, + index, + preparation: prepared, + slot, + sourceSequence: snapshot.sourceSequence, + }); + } + if (prepared.status !== "ready") { + cancelPrevious(); + return Object.freeze({ + reason: unavailableCompletionReason(prepared), + retryable: false, + revision: snapshot.revision, + status: "unavailable", + }); + } + const localSite = analyzeSqlLocalRelationSite( + prepared.statement, + position, + ); + if (localSite.status !== "ready") { + cancelPrevious(); + return Object.freeze({ + reason: unavailableCompletionReason(localSite), + retryable: false, + revision: snapshot.revision, + status: "unavailable", + }); + } + if ( + this.#activeCompletion !== active || + active.cancelReason !== null || + snapshot.revision !== this.#snapshot.revision + ) { + return completionCancellation( + snapshot.revision, + active.cancelReason ?? "superseded", + ); + } + + const querySite = localSite.querySite; + const statementOffset = + slot.boundaryQuality === "exact" ? slot.source.from : 0; + const replacementRange = Object.freeze({ + from: statementOffset + querySite.typedPathRange.from, + to: statementOffset + querySite.typedPathRange.to, + }); + const catalog = resolveCatalogContext(snapshot.context); + let catalogOutcome: SqlComposableCatalogOutcome = null; + let remainingIntentLeaseMs = 0; + const owner = this.#catalogOwner; + + if (catalog && this.#catalogCoordinator) { + if (!owner) { + cancelPrevious(); + catalogOutcome = Object.freeze({ + reason: "overloaded", + status: "unavailable", + }); + } else { + const ticket = owner.request({ + continuationToken: null, + limit: MAX_RELATION_COMPLETION_RESULTS, + prefix: querySite.prefix, + qualifier: querySite.qualifier, + searchPaths: catalog.searchPaths, + }); + if (this.#refreshIntent === previousIntent) { + this.#refreshIntent = null; + } + active.ticket = ticket; + const elapsed = Math.max( + 0, + performance.now() - completionStartedAt, + ); + const remaining = Math.max( + 0, + this.#catalogResponseBudgetMs - elapsed, + ); + let responseTimer: + | ReturnType + | undefined; + const raced = await Promise.race([ + ticket.result.then((outcome) => + Object.freeze({ + kind: "outcome" as const, + outcome, + }), + ), + new Promise<{ readonly kind: "timeout" }>((resolve) => { + responseTimer = setTimeout( + () => resolve(Object.freeze({ kind: "timeout" })), + remaining, + ); + }), + ]); + clearTimeout(responseTimer); + if ( + this.#activeCompletion !== active || + active.cancelReason !== null || + snapshot.revision !== this.#snapshot.revision + ) { + ticket.cancel(); + return completionCancellation( + snapshot.revision, + active.cancelReason ?? "superseded", + ); + } + let providerOutcome: SqlCatalogSearchWorkOutcome | null = + raced.kind === "outcome" ? raced.outcome : null; + if (raced.kind === "timeout") { + const retained = ticket.retainForRefresh( + (): (() => undefined) | null => + this.#prepareServiceChange( + "catalog-availability", + active, + ), + ); + if (retained.status === "retained") { + remainingIntentLeaseMs = retained.remainingLeaseMs; + this.#refreshIntent = active; + catalogOutcome = Object.freeze({ + status: "loading", + }); + } else if ( + retained.reason === "not-retainable" + ) { + providerOutcome = await ticket.result; + } else { + catalogOutcome = Object.freeze({ + reason: "execution-timeout", + status: "unavailable", + }); + } + } + if (providerOutcome) { + const outcome = providerOutcome; + if ( + outcome.status === "cancelled" || + outcome.status === "superseded" + ) { + return completionCancellation( + snapshot.revision, + active.cancelReason ?? + (outcome.status === "cancelled" + ? "caller" + : "superseded"), + ); + } + if (outcome.status === "unavailable") { + if ( + outcome.reason === "disposed" || + outcome.reason === "inactive" + ) { + return completionCancellation( + snapshot.revision, + this.#disposed ? "disposed" : "superseded", + ); + } + switch (outcome.reason) { + case "invalid-request": + catalogOutcome = Object.freeze({ + reason: "provider-failed", + status: "unavailable", + }); + break; + case "execution-timeout": + case "malformed-response": + case "overloaded": + case "provider-failed": + case "queue-timeout": + catalogOutcome = Object.freeze({ + reason: outcome.reason, + status: "unavailable", + }); + break; + } + } else { + catalogOutcome = outcome; + if (outcome.response.status === "loading") { + remainingIntentLeaseMs = + TERMINAL_LOADING_INTENT_LEASE_MS; + const cell: SessionTimerCell = { + active: true, + handle: undefined, + }; + this.#terminalIntentTimer = cell; + const handle = setTimeout(() => { + if (!cell.active) return; + cell.active = false; + if (this.#terminalIntentTimer === cell) { + this.#terminalIntentTimer = null; + } + }, remainingIntentLeaseMs); + cell.handle = handle; + if ( + !cell.active || + this.#terminalIntentTimer !== cell + ) { + clearTimeout(handle); + } + } + } + } + } + } else { + cancelPrevious(); + } + + const composition = composeSqlRelationCompletion({ + catalogOutcome, + dialect: snapshot.dialect.relationDialect, + localSite, + providerId: + catalog && this.#catalogCoordinator + ? this.#catalogCoordinator.providerId + : null, + remainingIntentLeaseMs, + replacementRange, + statementOffset, + }); + if ( + this.#activeCompletion !== active || + active.cancelReason !== null || + snapshot.revision !== this.#snapshot.revision + ) { + return completionCancellation( + snapshot.revision, + active.cancelReason ?? "superseded", + ); + } + return Object.freeze({ + revision: snapshot.revision, + sources: composition.sources, + status: "ready", + value: composition.value, + }); + } finally { + signal?.removeEventListener("abort", onAbort); + if (this.#activeCompletion === active) { + this.#activeCompletion = null; + } + } + }; + readonly update = (update: SqlDocumentUpdate): SqlRevision => { if (this.#disposed) { throw new SqlSessionError("session-disposed", "SQL document session is disposed"); @@ -866,6 +1602,13 @@ export class DefaultSqlDocumentSession nextContext, this.#dialects, ); + const nextCatalog = resolveCatalogContext(nextContext); + if (nextCatalog && !this.#catalogCoordinator) { + throw new SqlSessionError( + "invalid-context", + "SQL catalog context requires a configured catalog provider", + ); + } const nextLexicalProfile = nextDialect.lexicalProfile; if (this.#disposed) { throw new SqlSessionError( @@ -919,9 +1662,39 @@ export class DefaultSqlDocumentSession }) : null; } + const activeCompletion = this.#activeCompletion; + const refreshIntent = this.#refreshIntent; + const invalidatesLocalRelationCache = + nextSourceSequence !== this.#snapshot.sourceSequence || + nextDialect.relationDialect !== + this.#snapshot.dialect.relationDialect; this.#snapshot = nextSnapshot; this.#statementIndexCache = nextStatementIndexCache; - return revision; + if (invalidatesLocalRelationCache) { + this.#localRelationStatementCache = null; + } + this.#activeCompletion = null; + this.#refreshIntent = null; + this.#clearTerminalIntent(); + if ( + activeCompletion && + activeCompletion.cancelReason === null + ) { + activeCompletion.cancelReason = "superseded"; + } + if ( + refreshIntent && + refreshIntent !== activeCompletion && + refreshIntent.cancelReason === null + ) { + refreshIntent.cancelReason = "superseded"; + } + activeCompletion?.ticket?.cancel(); + if (refreshIntent !== activeCompletion) { + refreshIntent?.ticket?.cancel(); + } + this.#replaceCatalogOwner(); + return this.#snapshot.revision; } readonly isCurrent = (revision: SqlRevision): boolean => { @@ -933,7 +1706,34 @@ export class DefaultSqlDocumentSession return; } this.#disposed = true; + const activeCompletion = this.#activeCompletion; + const refreshIntent = this.#refreshIntent; + const catalogOwner = this.#catalogOwner; + this.#activeCompletion = null; + this.#refreshIntent = null; + this.#catalogOwner = null; + this.#clearTerminalIntent(); + this.#listeners.clear(); + this.#localRelationStatementCache = null; this.#statementIndexCache = null; + if ( + activeCompletion && + activeCompletion.cancelReason === null + ) { + activeCompletion.cancelReason = "disposed"; + } + if ( + refreshIntent && + refreshIntent !== activeCompletion && + refreshIntent.cancelReason === null + ) { + refreshIntent.cancelReason = "disposed"; + } + activeCompletion?.ticket?.cancel(); + if (refreshIntent !== activeCompletion) { + refreshIntent?.ticket?.cancel(); + } + catalogOwner?.dispose(); this.#onDispose(); }; } @@ -941,6 +1741,8 @@ export class DefaultSqlDocumentSession export class DefaultSqlLanguageService implements SqlLanguageService { + readonly #catalogCoordinator: SqlCatalogSearchWorkCoordinator | null; + readonly #completion: CompletionConfiguration; readonly #dialects: ReadonlyMap; readonly #sessions = new Set>(); #disposed = false; @@ -1001,6 +1803,78 @@ export class DefaultSqlLanguageService dialects.set(runtime.dialect.id, runtime); } this.#dialects = dialects; + + let catalogResponseBudgetMs = + DEFAULT_CATALOG_RESPONSE_BUDGET_MS; + const completion = readOwnDataProperty( + options, + "completion", + "invalid-service-options", + "SQL language service options", + ); + if (completion.found && completion.value !== undefined) { + if ( + completion.value === null || + typeof completion.value !== "object" + ) { + throw new SqlSessionError( + "invalid-service-options", + "SQL completion options must be an object", + ); + } + const budget = readOwnDataProperty( + completion.value, + "catalogResponseBudgetMs", + "invalid-service-options", + "SQL completion options", + ); + if (budget.found && budget.value !== undefined) { + if ( + typeof budget.value !== "number" || + !Number.isFinite(budget.value) || + budget.value < 0 || + budget.value > MAX_CATALOG_RESPONSE_BUDGET_MS + ) { + throw new SqlSessionError( + "invalid-service-options", + `Catalog response budget must be between 0 and ${MAX_CATALOG_RESPONSE_BUDGET_MS} milliseconds`, + ); + } + catalogResponseBudgetMs = budget.value; + } + } + this.#completion = Object.freeze({ + catalogResponseBudgetMs, + }); + + const catalog = readOwnDataProperty( + options, + "catalog", + "invalid-service-options", + "SQL language service options", + ); + if (!catalog.found || catalog.value === undefined) { + this.#catalogCoordinator = null; + } else { + const captured = captureSqlRelationCatalogProvider( + catalog.value, + ); + if (captured.status !== "accepted") { + throw new SqlSessionError( + "invalid-service-options", + "SQL relation catalog provider is invalid", + ); + } + const coordinator = + createSqlCatalogSearchWorkCoordinator(captured.value); + if (coordinator.status !== "created") { + throw new SqlSessionError( + "invalid-service-options", + "SQL relation catalog coordinator could not be created", + ); + } + this.#catalogCoordinator = coordinator.coordinator; + } } catch (error) { if (error instanceof SqlSessionError) { throw error; @@ -1062,12 +1936,21 @@ export class DefaultSqlLanguageService } const context = cloneContext(candidateContext); resolveDialectRuntime(context, this.#dialects); + const catalogContext = resolveCatalogContext(context); + if (catalogContext && !this.#catalogCoordinator) { + throw new SqlSessionError( + "invalid-context", + "SQL catalog context requires a configured catalog provider", + ); + } let session: DefaultSqlDocumentSession; session = new DefaultSqlDocumentSession( source, context, this.#dialects, + this.#catalogCoordinator, + this.#completion, () => { this.#sessions.delete(session); }, @@ -1107,6 +1990,7 @@ export class DefaultSqlLanguageService session.dispose(); } this.#sessions.clear(); + this.#catalogCoordinator?.dispose(); }; } diff --git a/src/vnext/types.ts b/src/vnext/types.ts index 8e732ac..18d11e8 100644 --- a/src/vnext/types.ts +++ b/src/vnext/types.ts @@ -136,6 +136,12 @@ export interface OpenSqlDocument { export interface SqlDocumentSession { readonly revision: SqlRevision; readonly update: (update: SqlDocumentUpdate) => SqlRevision; + readonly complete: ( + request: SqlCompletionRequest, + ) => Promise; + readonly onDidChange: ( + listener: (event: SqlSessionChangeEvent) => void, + ) => SqlDisposable; readonly isCurrent: (revision: SqlRevision) => boolean; readonly dispose: () => void; } @@ -149,6 +155,10 @@ export interface SqlLanguageService { } export interface SqlLanguageServiceOptions { + readonly catalog?: SqlRelationCatalogProvider | undefined; + readonly completion?: { + readonly catalogResponseBudgetMs?: number | undefined; + } | undefined; readonly dialects: readonly SqlDialect[]; } @@ -156,6 +166,7 @@ export type SqlSessionErrorCode = | "duplicate-dialect" | "invalid-change" | "invalid-context" + | "invalid-completion-request" | "invalid-dialect" | "invalid-document" | "invalid-service-options" @@ -174,3 +185,10 @@ export class SqlSessionError extends Error { this.code = code; } } +import type { + SqlCompletionRequest, + SqlDisposable, + SqlRelationCatalogProvider, + SqlCompletionResult, + SqlSessionChangeEvent, +} from "./relation-completion-types.js"; diff --git a/state/artifacts/reports/pr-197-browser-ci.md b/state/artifacts/reports/pr-197-browser-ci.md new file mode 100644 index 0000000..56da840 --- /dev/null +++ b/state/artifacts/reports/pr-197-browser-ci.md @@ -0,0 +1,44 @@ +# PR #197 browser CI triage + +## Investigation + +1. Inspected the failing browser job before forming a hypothesis. +2. Confirmed browser tests and worker builds were not the failing step. +3. Located the first failure in worker-placement bundle verification. +4. Compared the emitted bundle with the checked-in capability charter. + +## Root cause + +The framework-independent vNext bundle grew from the session walking skeleton +to the completed relation service and catalog scheduler. The exact local +packed-consumer measurement is 40,121 gzip/143,265 raw bytes; hosted Linux CI +measured 40,214 gzip bytes. It still contains no parser modules. + +The worker-placement script retained the obsolete 16 KiB skeleton limit, while +the capability charter already budgets 75 KiB for core plus the future +CodeMirror adapter. The complete worker fixture also embeds the page-side core, +so after the first correction its obsolete aggregate ceiling failed at 150,847 +gzip bytes. The exact local aggregate is 150,985 gzip/669,106 raw bytes; its +PostgreSQL and BigQuery lazy grammar closures are unchanged. + +## Fix + +Allocate 48 KiB gzip and 180 KiB raw to the framework-independent core. This +passes the current measured bundle with bounded headroom and reserves 27 KiB of +the declared compressed budget for the separate CodeMirror adapter. + +Allocate 160 KiB gzip and 700 KiB raw to the complete worker evidence fixture. +This is an aggregate test-fixture ceiling, not a product bundle promise: it +includes the page-side core plus both lazy parser grammars. The dedicated +dialect-closure ceilings remain unchanged and continue to detect parser growth. + +## Friction and future guidance + +The local isolated worktree shares dependencies by symlink, so its root install +must run with CI semantics to avoid an interactive module-directory prompt. +Hosted CI remains authoritative for platform-sensitive compressed sizes; the +local exact-tarball report records the raw sizes and dependency graph. + +Future vertical slices that change public entry-point reachability should +compare emitted bundle composition with both the per-entry allocation and the +combined 75 KiB capability budget. diff --git a/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts b/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts index 57ef37f..edc16c8 100644 --- a/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts +++ b/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts @@ -4,7 +4,7 @@ import type { SqlCompletionInfoResolver, SqlCompletionInfoResolverContext, } from "../../src/vnext/codemirror/relation-completion-types.js"; -import type { SqlRelationCompletionItem } from "../../src/vnext/relation-completion-types.js"; +import type { SqlCompletionItem } from "../../src/vnext/relation-completion-types.js"; interface ReactRootLike { readonly render: (value: unknown) => void; @@ -28,7 +28,7 @@ const resolveInfo: SqlCompletionInfoResolver = async (item, { signal }) => { }; }; -declare const item: SqlRelationCompletionItem; +declare const item: SqlCompletionItem; const resolved = resolveInfo(item, { signal: new AbortController().signal, }); @@ -42,7 +42,7 @@ if (item.provenance.kind === "catalog") { // @ts-expect-error live editor state is not a resolver parameter const resolverWithView: SqlCompletionInfoResolver = ( - _item: SqlRelationCompletionItem, + _item: SqlCompletionItem, _context: SqlCompletionInfoResolverContext, _view: EditorView, ) => null; diff --git a/test/vnext-types/marimo-relation-completion.test-d.ts b/test/vnext-types/marimo-relation-completion.test-d.ts index 721dfa4..9a8e74d 100644 --- a/test/vnext-types/marimo-relation-completion.test-d.ts +++ b/test/vnext-types/marimo-relation-completion.test-d.ts @@ -1,6 +1,7 @@ import type { SqlCatalogContext, SqlDocumentContext, + SqlDocumentSession, SqlDocumentEdit, SqlDocumentUpdate, SqlEmbeddedRegion, @@ -18,10 +19,9 @@ import type { SqlCompletionCancellationReason, SqlCompletionIssue, SqlDisposable, - SqlRelationCompletionItem, - SqlRelationCompletionList, + SqlCompletionItem, + SqlCompletionList, SqlRelationCatalogProvider, - SqlRelationCompletionSession, } from "../../src/vnext/relation-completion-types.js"; import type { SqlCatalogEpochTransitionTarget, @@ -60,7 +60,7 @@ const openWithRegions: OpenSqlDocument = { text: "SELECT * FROM {df}", }; -declare const session: SqlRelationCompletionSession; +declare const session: SqlDocumentSession; session.update({ baseRevision: session.revision, document: { kind: "replace", text: "SELECT * FROM {next_df}" }, @@ -325,19 +325,19 @@ const mismatchedItem = { providerId: "marimo", }, relationKind: "cte", -} satisfies SqlRelationCompletionItem; +} satisfies SqlCompletionItem; const contradictoryCompleteList = { isIncomplete: false, // @ts-expect-error complete lists cannot carry incomplete issues issues: [{ reason: "catalog-partial" }], items: [], -} satisfies SqlRelationCompletionList; +} satisfies SqlCompletionList; const contradictoryIncompleteList = { isIncomplete: true, issues: [], items: [], // @ts-expect-error incomplete lists require at least one issue -} satisfies SqlRelationCompletionList; +} satisfies SqlCompletionList; // @ts-expect-error timeouts are unavailable evidence, not cancellation const invalidCancellation: SqlCompletionCancellationReason = "timeout"; const undefinedContext: SqlDocumentUpdate = { diff --git a/test/vnext-types/session.test-d.ts b/test/vnext-types/session.test-d.ts index 13c7020..520056c 100644 --- a/test/vnext-types/session.test-d.ts +++ b/test/vnext-types/session.test-d.ts @@ -7,6 +7,7 @@ import { type SqlDocumentSession, type SqlEmbeddedRegion, type SqlLanguageService, + type SqlRelationCatalogProvider, type SqlRevision, type SqlTextChange, type SqlTextRange, @@ -33,6 +34,21 @@ void typedDialect.grammar; // @ts-expect-error dialect rendering policy is package-private void typedDialect.renderRelationPath; const service = createSqlLanguageService({ dialects: [dialect] }); +const relationCatalog = { + id: "host-catalog", + search: async () => ({ + coverage: { kind: "complete" as const }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready" as const, + }), +} satisfies SqlRelationCatalogProvider; +const catalogService = createSqlLanguageService({ + catalog: relationCatalog, + completion: { catalogResponseBudgetMs: 40 }, + dialects: [dialect], +}); +void catalogService; const session = service.openDocument({ context: { dialect: "duckdb", engine: "local" }, embeddedRegions: [{ from: 0, language: "python", to: 1 }], @@ -53,6 +69,25 @@ const hostOpen = { }; service.openDocument(hostOpen); const revision: SqlRevision = session.revision; +void session.complete({ + position: 0, + trigger: { kind: "invoked" }, +}); +void session.complete({ + position: 0, + signal: undefined, + trigger: { kind: "invoked" }, +}); +void session.complete({ + position: 0, + // @ts-expect-error invoked triggers cannot also carry a character + trigger: { character: ".", kind: "invoked" }, +}); +const changeSubscription = session.onDidChange((event) => { + const changedRevision: SqlRevision = event.revision; + void changedRevision; +}); +changeSubscription.dispose(); declare const maybeContext: HostContext | undefined; declare const maybeDocument: SqlDocumentEdit | undefined; diff --git a/test/worker-placement/README.md b/test/worker-placement/README.md index b1dbf6c..564c035 100644 --- a/test/worker-placement/README.md +++ b/test/worker-placement/README.md @@ -9,23 +9,23 @@ package's dependency. The fixture workspace pins Vite's floating transitive versions to the exact versions in the root lock, so the nested frozen install can run offline after a clean root CI install. -The minified Vite 8 packed-consumer baseline is 15,028 gzip/47,141 raw +The minified Vite 8 packed-consumer baseline is 40,121 gzip/143,265 raw bytes for the complete parser-free core, 67,573 gzip bytes for the PostgreSQL transitive graph, 50,470 gzip bytes for the BigQuery transitive graph, and -125,937 gzip/572,982 raw bytes for the complete worker build output. The core -measurement includes the four authenticated relation-dialect runtimes and -their reserved-word tables. The PostgreSQL and BigQuery figures each include -their transitive shared chunks; the report also identifies those shared chunks -explicitly. +150,985 gzip/669,106 raw bytes for the complete worker build output. The core +measurement includes the four authenticated relation-dialect runtimes, their +reserved-word tables, and relation-completion/session orchestration. The +PostgreSQL and BigQuery figures each include their transitive shared chunks; +the report also identifies those shared chunks explicitly. -The fail-closed ceilings retain approximately 9% gzip and 13% raw headroom for -the core, 4% gzip and 2% raw headroom for the complete worker output, and the -existing tight dialect-graph headroom: +The fail-closed ceilings retain approximately 18% gzip and 22% raw headroom +for the core, 8% gzip and 7% raw headroom for the complete worker output, and +the existing tight dialect-graph headroom: -- Complete parser-free core: 16 KiB gzip and 52 KiB raw +- Complete parser-free core: 48 KiB gzip and 180 KiB raw - PostgreSQL transitive graph: 68 KiB gzip - BigQuery transitive graph: 50 KiB gzip -- Complete worker build output: 128 KiB gzip and 570 KiB raw +- Complete worker build output: 160 KiB gzip and 700 KiB raw These are provisional placement limits, not product bundle promises. The orchestration script fails closed when they are exceeded, when the dialects no