diff --git a/docs/adr/0003-node-sql-parser-adapter.md b/docs/adr/0003-node-sql-parser-adapter.md index 3e60f1f..15f21c5 100644 --- a/docs/adr/0003-node-sql-parser-adapter.md +++ b/docs/adr/0003-node-sql-parser-adapter.md @@ -145,11 +145,11 @@ A direct object or one-element array is accepted. Zero or multiple roots are `failed/malformed-output`. The public normalized artifact exposes only its closed statement kind and full -statement-relative range. The backend AST is retained in a module-private -`WeakMap` keyed by the authenticated artifact. It is neither enumerable nor -returned through the syntax contract. Future relation extraction must decode -and validate backend nodes inside the adapter boundary rather than exposing -the raw AST to core features. +statement-relative range. The adapter consumes the backend AST immediately +through the bounded query-binding decoder and retains only an immutable, +backend-neutral binding model in a module-private `WeakMap` keyed by the +authenticated artifact. The raw AST is not retained, enumerable, or returned +through the syntax contract. ### Cancellation and execution placement @@ -201,7 +201,7 @@ This decision does not define: - Stable parser or provider APIs - Session cache keys or eviction - Document-level syntax diagnostics -- Semantic relation, scope, column, or type models +- Public semantic relation, scope, column, or type models - A worker protocol - Native DuckDB or remote validation providers - Dremio parsing diff --git a/docs/adr/0004-isolated-parser-execution.md b/docs/adr/0004-isolated-parser-execution.md index c5556cb..4312273 100644 --- a/docs/adr/0004-isolated-parser-execution.md +++ b/docs/adr/0004-isolated-parser-execution.md @@ -141,9 +141,10 @@ The initial request contains only: DuckDB uses the PostgreSQL grammar. Target-dialect policy stays in the main realm. -The initial response contains only one closed outcome: +Protocol v2 contains only one closed outcome: -- Parsed normalized statement kind +- Parsed normalized statement kind and a bounded query-binding DTO, or `null` + when the statement has no supported query-binding evidence - Syntax rejection - Bounded unsupported reason - Bounded failure code; retryability is derived from that code @@ -183,16 +184,17 @@ settles the active operation exactly once without exposing raw event data. Raw backend ASTs will not cross the worker boundary and the first protocol will not introduce remote AST handles or worker-local AST leases. -The first relation-completion slice is parser-independent under +The first relation-completion slice remains parser-independent under [ADR 0005](0005-parser-independent-relation-completion.md). Protocol v1 remains -syntax-only. Incomplete relation completion must not wait for worker startup, +historical and protocol v2 adds the internal query-binding DTO. Incomplete +relation completion must not wait for worker startup, parser acceptance, timeout, or recovery. -A future scope-dependent semantic slice will parse once and run adapter-owned -semantic decoders in the worker realm. It will move the closed protocol -atomically to v2 and return a bounded scoped IR, with normalized syntax and -semantic availability represented independently. It will not return flat -relation names, raw ASTs, source text, or absolute document ranges. +The scope decoder runs once in the worker realm immediately after parsing. +It returns query blocks, relation/alias bindings, persistent scopes, typed +visibility regions, independent coverage, and closed issues. It does not +return flat relation names, raw ASTs, source text, or absolute document ranges. +The host revalidates the complete DTO before accepting the response. Worker-local AST caching is deferred until profiling demonstrates that reparsing is material enough to justify leases, byte accounting, generation diff --git a/docs/adr/0006-query-binding-model.md b/docs/adr/0006-query-binding-model.md new file mode 100644 index 0000000..27eb783 --- /dev/null +++ b/docs/adr/0006-query-binding-model.md @@ -0,0 +1,103 @@ +# ADR 0006: Internal query binding model + +## Status + +Accepted for the vNext implementation. The model remains package-internal until a +column-completion vertical slice has validated it across the supported dialect +corpora. + +## Context + +Column completion needs more than a parser's flat table list. It must distinguish +query blocks, relation declarations, aliases, correlation, and clause-specific +visibility. The parser adapter also has strict trust boundaries: raw parser ASTs +are private, parser results can be partial, and cached analysis must belong to +the exact statement and parser authority that produced it. + +The model must remain cheap enough for interactive use and must not make the +existing parser-independent relation completion path wait for parsing. + +## Decision + +Add an internal, backend-neutral query binding model with: + +- statement-relative UTF-16 ranges; +- query blocks and relation bindings identified by model-local array indexes; +- a persistent scope chain in which each node adds at most one binding; +- ordered, globally non-overlapping visibility regions that point to a scope; +- independent query-block, relation-binding, and visibility coverage; +- closed issue and unknown-source reason sets; +- private authentication and exact statement/parser-authority ownership. + +Globally non-overlapping regions deliberately exclude nested query text from +their parent region. A decoder splits a parent region around nested queries. +This permits binary-search lookup without duplicating the visible binding set. +Persistent scopes then reconstruct bindings in declaration order using linear +space. In particular: + +- a `SELECT` list can point to the completed `FROM` scope even though it appears + earlier in the source; +- a join condition points to the scope containing prior relations and its + current right-hand relation; +- a correlated child block can enter through a parent scope; +- an ordinary derived table enters through a scope that excludes same-level + siblings; a future proven `LATERAL` decoder may opt into that scope. + +Aliases are the visible qualifier when present and hide a named relation's base +name. Identifier equality is supplied by the dialect runtime; the model does +not lowercase or otherwise reinterpret identifiers. + +Construction accepts untrusted plain data and validates it before issuing an +immutable authenticated model. It rejects accessors, sparse or oversized +arrays, malformed UTF-16 identifiers, invalid or unrelated ranges, forward +parent edges, repeated bindings in a scope chain, overlapping regions, and +unknown bindings paired with complete relation coverage. + +Resolution returns: + +- a visible binding list with complete or partial coverage; +- one resolved binding, known ambiguity, or authoritative no-match; +- unavailable instead of no-match when coverage is partial. + +## Ownership and invalidation + +The future parser service owns bounded parse/model caching and in-flight +deduplication. A document session owns the mapping from its current statement +slots to immutable models. The cache key includes exact statement text, +dialect, parser authority/conformance, and implementation version. +Statement-relative coordinates allow an unchanged statement to be reused after +an edit shifts its absolute document position. Source, dialect, or parser +authority changes invalidate the model; catalog epoch and completion context +changes do not. + +The interactive statement limit is initially 16 KiB. The model also caps query +blocks at 256, relation bindings at 1,024, scopes and regions at 2,048 each, +nesting by the block limit, issues at 256, path depth at 8, and identifier +components at 256 UTF-16 code units. + +## Degradation + +A decoder may claim complete evidence only for directly conforming parser output +and constructs it understands. Compatibility parses can supply useful partial +evidence but cannot prove absence. Unknown AST subtrees become an unknown source +or a closed issue and downgrade the relevant coverage; they are never silently +dropped under complete coverage. + +Invalid, failed, opaque, incomplete, or resource-limited analysis becomes +partial or unavailable. Parser-independent relation completion remains +available. Dremio remains partial or unavailable until it has an owned semantic +corpus. + +## Consequences + +The model can support lazy batched column loading without exposing a parser AST +or CodeMirror's eager `SQLNamespace`. Its indexes are not stable identities +across revisions. Projection inference, output columns, star expansion, types, +expression binding, semantic diagnostics, navigation, rename, formatting, and +vendor constructs such as `PIVOT`, `UNNEST`, table functions, and unproven +`LATERAL` semantics are explicitly outside this slice. + +The node-sql-parser integration now supplies a strict AST-to-plain-data decoder, +normalizes inside the worker, and checks direct-versus-worker parity. Public +semantic APIs remain deferred until a column-completion vertical slice proves +the model against consumer and dialect corpora. diff --git a/docs/vnext/capability-charter.md b/docs/vnext/capability-charter.md index d04c8dd..68e5fe5 100644 --- a/docs/vnext/capability-charter.md +++ b/docs/vnext/capability-charter.md @@ -239,7 +239,8 @@ count and estimated retained bytes. Provisional compressed bundle budgets: -- Framework-independent core with relation completion: 48 KiB +- Framework-independent core with relation, namespace, and column completion: + 50 KiB - Core plus CodeMirror adapter, excluding parser and dialect data: 75 KiB - Each ordinary dialect module: 25 KiB - Optional `node-sql-parser` chunk: no regression from its recorded baseline diff --git a/docs/vnext/codemirror-adapter.md b/docs/vnext/codemirror-adapter.md index 572d52a..d721701 100644 --- a/docs/vnext/codemirror-adapter.md +++ b/docs/vnext/codemirror-adapter.md @@ -70,6 +70,51 @@ const support = sqlEditor({ }); ``` +## Statement boundaries and gutter + +The support object exposes synchronous structural queries without exposing the +adapter-owned session: + +```ts +const current = support.statementBoundaryAt(view, { + affinity: "left", + position: view.state.selection.main.head, +}); +const visible = support.statementBoundariesIntersecting(view, { + from: view.viewport.from, + to: view.viewport.to, +}); +``` + +Both methods return `null` for foreign, destroyed, or unsynchronized views. +Catalog-backed hosts can invalidate the adapter-owned session without exposing +it: + +```ts +const revision = support.invalidateCatalog(view); +``` + +This returns `null` for foreign or destroyed views. A live view advances its +revision, cancels stale catalog work, and forces relation, column, and namespace +providers to be consulted again. + +An opt-in structural gutter marks only lines intersecting scanner-owned SQL +`code` spans. It never parses or copies statement text: + +```ts +const support = sqlEditor({ + initialContext, + service, + statementGutter: { + hideWhenNotFocused: true, + showInactive: true, + }, +}); +``` + +The gutter uses `--cm-sql-statement-color` and +`--cm-sql-statement-inactive-opacity` CSS variables. It is disabled by default. + ## Atomic inputs Context and document changes can share one CodeMirror transaction: diff --git a/docs/vnext/column-catalog-authority.md b/docs/vnext/column-catalog-authority.md new file mode 100644 index 0000000..aff30c0 --- /dev/null +++ b/docs/vnext/column-catalog-authority.md @@ -0,0 +1,65 @@ +# vNext Column Catalog Authority + +Status: internal vertical-slice contract + +Column discovery is lazy, provider-owned, and batched. A completion request +sends at most 64 unresolved relation references in one provider invocation. +The deterministic first batch is useful but explicitly partial when more +visible relations match; completion reports `query-binding-partial` and never +claims the omitted relations were searched. Each reference carries a +caller-local `requestKey`, a decoded identifier path, and no unauthenticated +entity identity. The provider resolves paths against the supplied catalog +scope, search paths, and dialect. + +The provider returns stable relation and column entity IDs. Every accepted +column contains: + +- a canonical `SqlIdentifierComponent`; +- bounded provider-rendered `insertText`; +- a stable column entity ID and ordinal; and +- immutable provenance containing provider, scope, epoch, relation, and column + identities. + +Responses describe each requested relation independently as: + +- `ready` with complete or partial column coverage; +- `loading`; or +- `failed` with a normalized code and retry policy. + +Missing, extra, conflicting, oversized, accessor-backed, or malformed data is +rejected at the provider boundary. Relations and columns have deterministic +code-unit order. Duplicate request keys and conflicting stable IDs fail closed; +identical duplicate columns are deduplicated. + +## Epoch and cache behavior + +Cold requests use `expectedEpoch: null`. A response always supplies an epoch. +An owner remembers that observation, and later null-epoch requests reuse only +cache entries from the observed epoch. Explicit expected epochs never reuse +entries from another epoch. The cache is LRU-bounded by relation entries. +Only complete ready results are cached. Partial, loading, and failed results +remain visible to the caller but are eligible for another provider request. + +Relation-catalog subscription events invalidate the session's relation, column, +and namespace observations together. Hosts without a subscribed relation +provider call `session.invalidateCatalog()` when schema authority changes. The +method advances the session revision, cancels retained completion work, rebuilds +all catalog owners, and emits one `catalog` change event. + +## Lifecycle + +One owner represents one document/session authority for a scope and dialect. +Starting a newer request supersedes and aborts that owner's prior request. +Explicit cancellation settles as cancelled. Owner or coordinator disposal +aborts outstanding work and settles it as unavailable/disposed. Provider +rejections, throws, and malformed responses are contained; late settlements +cannot publish or populate the cache. + +The coordinator batches a request once, never once per relation. Cache hits and +misses are composed deterministically while only misses are sent to the +provider. + +Provider-declared loading receives at most one automatic retry for the same +document, context, and completion position. A persistent loading response is +then returned without a refresh token; hosts resume it through catalog +invalidation instead of an unbounded polling loop. diff --git a/docs/vnext/marimo-sql-migration.md b/docs/vnext/marimo-sql-migration.md new file mode 100644 index 0000000..77d17cc --- /dev/null +++ b/docs/vnext/marimo-sql-migration.md @@ -0,0 +1,202 @@ +# Marimo SQL Completion Migration + +Status: implementation fixture; relation, column, and namespace completion + +The compile-only fixture +[`marimo-sql-migration.test-d.ts`](../../test/vnext-types/marimo-sql-migration.test-d.ts) +models the intended marimo integration exclusively through public vNext +exports. It is not application code and has no runtime dependency on marimo. + +## Ownership and provider seam + +Marimo creates one caller-owned `SqlLanguageService` for an editor fleet. +Every `sqlEditor` support/view opens its own session against that service. +Destroying a view never disposes the service; the application disposes the +service after its views have been retired. + +One relation provider projects `dataConnectionsMapAtom` and +`datasetTablesAtom` into canonical relation records. It must preserve today's +collision rule: +connection relations win over same-named local relations. Provider records +contain plain immutable data and stable entity IDs, never Jotai atoms, React +roots, or backend objects. + +Catalog `scope` identifies a live connection incarnation: + +```text +: +``` + +Replacing or reconnecting a same-named connection creates a new incarnation +and scope. Schema/table mutations within an incarnation advance its provider +epoch and notify the one scoped subscription. The context `searchPath` values +reproduce +`default_database`, `default_schema`, nested-schema, and schemaless behavior. + +## Batched DataTable columns + +The same shared service configures one `SqlColumnCatalogProvider`. Each +`loadColumns` call forwards the complete relation request array to one +DataTable metadata batch operation. It does not fetch once per relation. +The column authority resolves each canonical relation path within the same +scope and search paths as relation completion. Column IDs and returned relation +IDs are stable within that connection incarnation. + +The library sends at most 64 matching physical relations in one request. +Larger visible sets are deterministically truncated and reported as partial; +the provider must not interpret one batch as exhaustive when completion carries +`query-binding-partial`. + +Each column supplies a canonical identifier and provider-rendered +`insertText`; these are intentionally separate so quoted or dialect-sensitive +names insert correctly. The provider also supplies ordinal, data type, and +detail when known. A cold request carries `expectedEpoch: null`. The returned +epoch becomes the authority for later work, while connection replacement gets +a new scope. + +Every requested relation independently reports complete or partial ready +columns, loading, or normalized failure. Partial, loading, and failed results +remain retryable according to the library contract and are not disguised as +complete empty schemas. The public boundary validates the provider's unknown +payload before it reaches completion composition. + +## One atomic editor input + +Marimo should have one transaction builder for SQL interpretation changes. A +document or engine change emits all of these in the same CodeMirror +transaction: + +1. marimo language metadata; +2. `support.contextEffect` with engine, registered vNext dialect, connection + scope, and search paths; +3. the CodeMirror SQL dialect compartment reconfiguration; and +4. `support.embeddedRegionsEffect` containing the complete region set in the + resulting document. + +No update listener should dispatch a follow-up context or dialect transaction. +That creates an observable grammar/catalog mismatch and makes rapid +connection switches race. + +## Templates and external sources + +`sqlEditor` owns the single `autocompletion()` configuration. Marimo passes its +Python variable and keyword sources through `autocomplete.externalSources`. +The region scanner recognizes single `{python}` expressions, ignores escaped +`{{`, includes both delimiters in each region, and supplies a complete sorted +set after every document change. SQL completion is unavailable inside those +regions while the external variable source remains active. + +Embedded regions are non-empty half-open ranges, so an unmatched `{` ending at +the document boundary cannot contain the insertion point at `doc.length`. +Marimo closes that boundary with +`autocomplete.isCompletionPositionAllowed`. Its synchronous scanner returns +false while the insertion point is in an unmatched Python expression. The +adapter then skips the SQL session source, while still running the installed +variable and keyword sources. A false result or thrown gate is fail-closed, +and a gate transition to false cancels owned SQL completion work and disposes +owned rich-info resources. + +The gate receives only the immutable CodeMirror `EditorState` and numeric +position. It must remain synchronous and side-effect free. Do not encode a +range beyond the document or an empty embedded region. + +The rich-info resolver uses relation `catalog` provenance or +`column-catalog` scope, relation ID, and column ID to look up current marimo +metadata. It mounts React into a new element and returns +`{ dom, destroy: root.unmount }`. The adapter owns cancellation and disposal. + +## Dialect cutover + +The first cutover is limited to the registered vNext dialects: + +- DuckDB +- PostgreSQL +- BigQuery +- Dremio + +MySQL, MariaDB, SQLite, MSSQL, Oracle, and standard/fallback engines continue +to use the legacy schema completion source. The router must select exactly one +table/schema source; installing legacy schema completion as an external source +for a vNext-enabled dialect would create duplicate and conflicting relation +edits. + +Full removal of the legacy source requires equivalent dialect handles or an +explicit, tested generic-dialect policy. + +## Remaining library gaps + +Marimo's current `tablesCompletionSource()` is broader than its name. The +CodeMirror SQL schema source provides relations, namespace navigation, and +columns. vNext now provides all three through separate bounded providers, +including qualified and unqualified query-site completion, ambiguity handling, +stable provenance, cancellation, and batched column work. + +The remaining feature gaps are: + +- the dialect coverage described above; and +- output-column inference for CTEs and derived relations. vNext completes CTE + relation names but deliberately does not send a visible CTE name to the + physical column provider. Keep the legacy column source for those sites, or + defer full source replacement, until projection/output-column inference + lands. + +The fixture feeds marimo's immutable namespace projection—stable entity ID, +scope, canonical identifier path, and namespace kind—through one public, +scoped namespace provider on the shared service. + +## Migration sequence + +1. Capture golden legacy results for labels, kinds, edit ranges, qualification, + and details across representative connection shapes. +2. Land the shared relation and batched column providers, incarnation scope, + atomic transaction builder, region scanner, insertion-point completion + gate, and completion router behind a feature flag. +3. Run vNext relation completion in shadow mode while the legacy source remains + visible. +4. Compare relation and column results with the golden corpus, including + quoted insert text, aliases, ambiguity, partial/loading/failure states, and + cold epoch behavior. +5. Cut over supported physical-relation sites while preserving variable, + keyword, and legacy CTE/derived-output column sources. +6. Add dialect coverage, expand the router, and remove completion-only legacy + schema code. Keep legacy schema data while hover or diagnostics still use + it. + +## Acceptance tests + +The compile-only marimo fixture proves: + +- one shared service configured with one relation provider, one batched + column provider, and one namespace provider; +- a two-relation cold column request with `expectedEpoch: null`; +- stable relation and column IDs, canonical identifiers, and distinct + provider-rendered insert text; +- partial, loading, and failure provider states; +- relation and column provenance lookup by current scope with a React + disposer; +- atomic editor interpretation updates, complete template regions, and the + unmatched-expression insertion-point gate; +- preservation of variable and keyword sources; and +- supported-dialect vNext routing with exclusive legacy fallback. + +Library tests cover relation and physical-column completion for `FROM`, `JOIN`, +`alias.`, unqualified projections and predicates, `USING`, correlated nested +queries, quoted identifiers, ambiguous columns, provider +loading/invalidations, bounded batching, and template barriers. CTE tests prove +declaration-order relation visibility and that CTE names are not incorrectly +resolved through the physical column provider; they do not prove CTE +output-column inference. + +Marimo integration tests cover: + +- default, nested, and schemaless database/schema layouts; +- local-table collisions where the connection relation wins; +- atomic engine/dialect/scope changes and rapid A-B-A switching; +- reconnecting under the same display name without stale cache reuse; +- open-menu schema invalidation; +- closed, escaped, and unmatched Python expressions; +- completion edits mapping regions in the same transaction; +- variable and keyword source preservation; +- React table/column info unmount on navigation, edit, and view destruction; +- one shared service across 1, 10, and 50 views; and +- exclusive legacy routing for unsupported dialects. diff --git a/docs/vnext/namespace-catalog-authority.md b/docs/vnext/namespace-catalog-authority.md new file mode 100644 index 0000000..500f81e --- /dev/null +++ b/docs/vnext/namespace-catalog-authority.md @@ -0,0 +1,45 @@ +# vNext Namespace Catalog Authority + +Status: public provider and session integration + +Namespace completion searches catalog, schema, project, and dataset containers. +One query site produces one bounded provider request containing its qualifier, +prefix, search paths, result limit, dialect, scope, and expected catalog epoch. +The API does not recursively walk a provider tree or issue one request per +container. + +Each result has a stable container entity ID, a canonical role-tagged identifier +path, provider-rendered insertion text, match quality, and immutable provenance. +The hostile boundary rejects accessors, sparse arrays, extra properties, +oversized data, invalid epochs, and conflicting duplicate identities. Identical +duplicate identities are collapsed and results receive deterministic code-unit +ordering. + +The composer applies the dialect's prefix matcher, deduplicates by authority +identity, and produces deterministic completion edits. Unknown or throwing +prefix comparison is reported as incomplete instead of guessing. + +## Epoch, cache, and lifecycle + +A cold request uses `expectedEpoch: null`; the returned epoch becomes the +owner's observed epoch. Complete ready searches are held in a bounded LRU keyed +by provider instance, scope, dialect, epoch, query site, search paths, and limit. +Partial, loading, and failed responses remain visible but are never cached. + +A newer request supersedes and aborts prior work for the same owner. Explicit +cancellation and owner/coordinator disposal abort pending work. Provider throws, +rejections, malformed data, and late settlements are contained. + +Session integration prepares one owner for each live scope/dialect and +disposes it when catalog authority changes. Query-site integration calls +`prepareSqlNamespaceCatalogSearch`, submits the result through the owner, and +passes the outcome plus the site's replacement range and dialect prefix matcher +to `composeSqlNamespaceCompletion`. Namespace items are merged with local and +relation-catalog items under the same bounded completion response budget. +Relation-catalog subscription events invalidate namespace observations. +Namespace-only hosts, and hosts whose relation provider has no subscription, +call `session.invalidateCatalog()` after a schema-authority change. + +Provider-declared loading receives at most one automatic retry for a stable +document, context, and completion position. Further loading responses do not +schedule polling; a host-provided catalog invalidation resumes discovery. diff --git a/scripts/worker-placement.mjs b/scripts/worker-placement.mjs index fb6e692..e453071 100644 --- a/scripts/worker-placement.mjs +++ b/scripts/worker-placement.mjs @@ -35,8 +35,8 @@ const PARSER_MARKERS = [ "tableList", ]; const BIGQUERY_GZIP_LIMIT = 50 * 1024; -const CORE_TOTAL_GZIP_LIMIT = 48 * 1024; -const CORE_TOTAL_RAW_LIMIT = 180 * 1024; +const CORE_TOTAL_GZIP_LIMIT = 50 * 1024; +const CORE_TOTAL_RAW_LIMIT = 184 * 1024; const POSTGRESQL_GZIP_LIMIT = 68 * 1024; const WORKER_TOTAL_GZIP_LIMIT = 160 * 1024; const WORKER_TOTAL_RAW_LIMIT = 700 * 1024; diff --git a/src/vnext/__tests__/column-catalog-batch-coordinator.bench.ts b/src/vnext/__tests__/column-catalog-batch-coordinator.bench.ts new file mode 100644 index 0000000..e585907 --- /dev/null +++ b/src/vnext/__tests__/column-catalog-batch-coordinator.bench.ts @@ -0,0 +1,101 @@ +import { bench, describe } from "vitest"; +import { + createSqlColumnCatalogBatchCoordinator, +} from "../column-catalog-batch-coordinator.js"; +import type { + SqlColumnCatalogBatchRequest, +} from "../column-catalog-types.js"; + +const epoch = Object.freeze({ generation: 1, token: "bench" }); +const relations = Object.freeze( + Array.from({ length: 64 }, (_, index) => + Object.freeze({ + path: Object.freeze([ + Object.freeze({ + quoted: false, + value: `relation_${index}`, + }), + ]), + requestKey: `relation_${index}`, + }) + ), +); +const searchPaths = Object.freeze([ + Object.freeze([ + Object.freeze({ quoted: false, value: "main" }), + ]), +]); + +function response(request: SqlColumnCatalogBatchRequest) { + return Object.freeze({ + epoch, + relations: Object.freeze(request.relations.map((relation) => + Object.freeze({ + columns: Object.freeze( + Array.from({ length: 64 }, (_, ordinal) => + Object.freeze({ + columnEntityId: + `${relation.requestKey}:column:${ordinal}`, + identifier: Object.freeze({ + quoted: false, + value: `column_${ordinal}`, + }), + insertText: `column_${ordinal}`, + ordinal, + }) + ), + ), + coverage: "complete", + relationEntityId: `entity:${relation.requestKey}`, + requestKey: relation.requestKey, + status: "ready", + }) + )), + }); +} + +function owner() { + const created = createSqlColumnCatalogBatchCoordinator({ + maxCacheEntries: 128, + provider: { + id: "benchmark", + loadColumns: (request: SqlColumnCatalogBatchRequest) => + Promise.resolve(response(request)), + }, + }); + if (created.status !== "created") throw new Error("coordinator"); + const prepared = created.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "benchmark", + }); + if (prepared.status !== "prepared") throw new Error("owner"); + return prepared.owner; +} + +describe("column catalog batch coordinator", () => { + bench("cold 64 relation x 64 column batch", async () => { + const current = owner(); + await current.request({ + expectedEpoch: null, + relations, + searchPaths, + }).result; + current.dispose(); + }); + + const warmOwner = owner(); + const warmReady = warmOwner.request({ + expectedEpoch: epoch, + relations, + searchPaths, + }).result; + + bench("warm 64 relation cache projection", async () => { + await warmReady; + await warmOwner.request({ + expectedEpoch: epoch, + relations, + searchPaths, + }).result; + }); +}); diff --git a/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts new file mode 100644 index 0000000..97fb1c4 --- /dev/null +++ b/src/vnext/__tests__/column-catalog-batch-coordinator.test.ts @@ -0,0 +1,593 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createSqlColumnCatalogBatchCoordinator, +} from "../column-catalog-batch-coordinator.js"; +import type { + SqlColumnCatalogBatchRequest, +} from "../column-catalog-types.js"; +import type { SqlCatalogEpoch } from "../relation-completion-types.js"; + +const epoch: SqlCatalogEpoch = + Object.freeze({ generation: 1, token: "one" }); +const nextEpoch: SqlCatalogEpoch = + Object.freeze({ generation: 2, token: "two" }); + +function reference(requestKey: string, name = requestKey) { + return Object.freeze({ + path: Object.freeze([ + Object.freeze({ quoted: false, value: name }), + ]), + requestKey, + }); +} + +function ready( + request: SqlColumnCatalogBatchRequest, + suffix = "", +) { + return { + epoch: request.expectedEpoch ?? epoch, + relations: request.relations.map((relation) => ({ + columns: [{ + columnEntityId: `column-${relation.requestKey}${suffix}`, + identifier: { + quoted: false, + value: `value_${relation.requestKey}${suffix}`, + }, + insertText: `value_${relation.requestKey}${suffix}`, + ordinal: 0, + }], + coverage: "complete", + relationEntityId: `relation-${relation.requestKey}`, + requestKey: relation.requestKey, + status: "ready", + })), + }; +} + +function setup( + loadColumns: ( + request: SqlColumnCatalogBatchRequest, + signal: AbortSignal, + ) => unknown, + maxCacheEntries = 32, +) { + const result = createSqlColumnCatalogBatchCoordinator({ + maxCacheEntries, + provider: { id: "catalog", loadColumns }, + }); + if (result.status !== "created") { + throw new Error("Expected coordinator"); + } + const prepared = result.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + }); + if (prepared.status !== "prepared") { + throw new Error("Expected owner"); + } + return { + coordinator: result.coordinator, + owner: prepared.owner, + }; +} + +function input( + relations = [reference("users"), reference("events")], + epoch_: SqlCatalogEpoch | null = epoch, +) { + return { + expectedEpoch: epoch_, + relations, + searchPaths: [[{ quoted: false, value: "main" }]], + }; +} + +function deferred() { + let resolve: (value: Value) => void = (): void => {}; + let reject: (reason?: unknown) => void = (): void => {}; + const promise = new Promise((resolve_, reject_) => { + resolve = resolve_; + reject = reject_; + }); + return { promise, reject, resolve }; +} + +describe("column catalog batch coordinator", () => { + it("loads every missing relation in one deterministic batch", async () => { + const requests: SqlColumnCatalogBatchRequest[] = []; + const { coordinator, owner } = setup((request) => { + requests.push(request); + return ready(request); + }); + const outcome = await owner.request(input([ + reference("users"), + reference("events"), + reference("accounts"), + ])).result; + + expect(requests).toHaveLength(1); + expect(requests[0]?.relations.map((relation) => + relation.requestKey + )).toEqual(["accounts", "events", "users"]); + expect(outcome).toMatchObject({ + providerId: "catalog", + relations: [ + { requestKey: "accounts", status: "ready" }, + { requestKey: "events", status: "ready" }, + { requestKey: "users", status: "ready" }, + ], + status: "usable", + }); + expect(Object.isFrozen(outcome)).toBe(true); + if (outcome.status === "usable") { + expect(Object.isFrozen(outcome.relations)).toBe(true); + } + coordinator.dispose(); + }); + + it("caches ready relations by authority identity and remaps request keys", async () => { + const requests: SqlColumnCatalogBatchRequest[] = []; + const { owner } = setup((request) => { + requests.push(request); + return ready(request); + }); + await owner.request(input([reference("first", "users")])).result; + const cached = await owner.request(input([ + reference("renamed", "users"), + ])).result; + await owner.request(input( + [reference("new-epoch", "users")], + nextEpoch, + )).result; + + expect(requests).toHaveLength(2); + expect(cached).toMatchObject({ + relations: [{ + relationEntityId: "relation-first", + requestKey: "renamed", + }], + status: "usable", + }); + }); + + it("separates quoted relation and search-path cache identities", async () => { + let calls = 0; + const { owner } = setup((request) => { + calls += 1; + return ready(request); + }); + const quoted = { + expectedEpoch: epoch, + relations: [{ + path: [{ quoted: true, value: "users" }], + requestKey: "quoted", + }], + searchPaths: [[{ quoted: true, value: "main" }]], + }; + + await expect(owner.request(quoted).result).resolves.toMatchObject({ + status: "usable", + }); + await expect(owner.request(quoted).result).resolves.toMatchObject({ + status: "usable", + }); + expect(calls).toBe(1); + }); + + it("supports cold null epochs and reuses the observed response epoch", async () => { + const expected: Array = []; + const { owner } = setup((request) => { + expected.push(request.expectedEpoch); + return { + ...ready(request), + epoch, + }; + }); + await owner.request(input([reference("users")], null)).result; + const cached = await owner.request( + input([reference("renamed", "users")], null), + ).result; + + expect(expected).toEqual([null]); + expect(cached).toMatchObject({ + relations: [{ requestKey: "renamed" }], + status: "usable", + }); + }); + + it("combines cached and loaded relations without changing order", async () => { + const requests: SqlColumnCatalogBatchRequest[] = []; + const { owner } = setup((request) => { + requests.push(request); + return ready(request); + }); + await owner.request(input([reference("users")])).result; + const mixed = await owner.request(input([ + reference("users"), + reference("events"), + ])).result; + + expect(requests).toHaveLength(2); + expect(requests[1]?.relations.map((relation) => + relation.requestKey + )).toEqual(["events"]); + expect(mixed).toMatchObject({ + relations: [ + { requestKey: "events" }, + { requestKey: "users" }, + ], + status: "usable", + }); + }); + + it("caches only complete ready coverage", async () => { + let calls = 0; + const { owner } = setup((request) => { + calls += 1; + return { + epoch: request.expectedEpoch ?? epoch, + relations: request.relations.map((relation, index) => + index === 0 + ? { requestKey: relation.requestKey, status: "loading" } + : index === 1 + ? { + columns: [], + coverage: "partial", + relationEntityId: `relation-${relation.requestKey}`, + requestKey: relation.requestKey, + status: "ready", + } + : { + code: "unavailable", + requestKey: relation.requestKey, + retry: "next-request", + status: "failed", + } + ), + }; + }); + const uncached = [ + reference("loading"), + reference("partial"), + reference("failed"), + ]; + const first = await owner.request(input(uncached)).result; + const second = await owner.request(input(uncached)).result; + + expect(first).toMatchObject({ + relations: [ + { requestKey: "failed", status: "loading" }, + { requestKey: "loading", status: "ready" }, + { requestKey: "partial", status: "failed" }, + ], + status: "usable", + }); + expect(second.status).toBe("usable"); + expect(calls).toBe(2); + }); + + it("aborts and settles cancelled work exactly once", async () => { + const work = deferred(); + const signals: AbortSignal[] = []; + const { owner } = setup((_request, signal_) => { + signals.push(signal_); + return work.promise; + }); + const ticket = owner.request(input()); + ticket.cancel(); + ticket.cancel(); + + await expect(ticket.result).resolves.toEqual({ status: "cancelled" }); + expect(signals[0]?.aborted).toBe(true); + work.resolve({}); + await Promise.resolve(); + await expect(ticket.result).resolves.toEqual({ status: "cancelled" }); + }); + + it("ignores a provider rejection after its consumer is cancelled", async () => { + const work = deferred(); + const { owner } = setup(() => work.promise); + const ticket = owner.request(input()); + + ticket.cancel(); + work.reject(new Error("late rejection")); + await expect(ticket.result).resolves.toEqual({ status: "cancelled" }); + await Promise.resolve(); + }); + + it("supersedes prior owner work but isolates other owners", async () => { + const pending: Array>> = []; + const { coordinator, owner } = setup(() => { + const work = deferred(); + pending.push(work); + return work.promise; + }); + const otherResult = coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + }); + if (otherResult.status !== "prepared") { + throw new Error("Expected second owner"); + } + const first = owner.request(input()); + const other = otherResult.owner.request(input()); + const second = owner.request(input([reference("next")])); + + await expect(first.result).resolves.toEqual({ + status: "superseded", + }); + let otherSettled = false; + void other.result.then(() => { + otherSettled = true; + }); + await Promise.resolve(); + expect(otherSettled).toBe(false); + second.cancel(); + other.cancel(); + }); + + it("preserves the newest request across reentrant abort handlers", async () => { + const signals: AbortSignal[] = []; + let owner: ReturnType["owner"] | null = null; + const reentrant: { + ticket: + | ReturnType["owner"]["request"]> + | null; + } = { ticket: null }; + const configured = setup((_request, signal) => { + signals.push(signal); + if (signals.length === 1) { + signal.addEventListener("abort", () => { + reentrant.ticket = + owner?.request(input([reference("reentrant")])) ?? null; + }, { once: true }); + } + return new Promise(() => undefined); + }); + owner = configured.owner; + const first = owner.request(input([reference("first")])); + const interrupted = owner.request(input([reference("interrupted")])); + + await expect(first.result).resolves.toEqual({ + status: "superseded", + }); + await expect(interrupted.result).resolves.toEqual({ + status: "superseded", + }); + expect(signals).toHaveLength(2); + expect(signals.map((signal) => signal.aborted)).toEqual([ + true, + false, + ]); + + const newest = owner.request(input([reference("newest")])); + expect(signals.map((signal) => signal.aborted)).toEqual([ + true, + true, + false, + ]); + await expect(reentrant.ticket?.result).resolves.toEqual({ + status: "superseded", + }); + newest.cancel(); + }); + + it.each(["owner", "coordinator"] as const)( + "does not resurrect work after reentrant %s disposal", + async (target) => { + const signals: AbortSignal[] = []; + let dispose = (): void => {}; + const configured = setup((_request, signal) => { + signals.push(signal); + if (signals.length === 1) { + signal.addEventListener("abort", () => dispose(), { + once: true, + }); + } + return new Promise(() => undefined); + }); + dispose = target === "owner" + ? configured.owner.dispose + : configured.coordinator.dispose; + const first = configured.owner.request( + input([reference("first")]), + ); + const interrupted = configured.owner.request( + input([reference("interrupted")]), + ); + + await expect(first.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + await expect(interrupted.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + expect(signals).toHaveLength(1); + expect(signals[0]?.aborted).toBe(true); + }, + ); + + it("disposes owners and coordinator with prompt aborts", async () => { + const signals: AbortSignal[] = []; + const { coordinator, owner } = setup((_request, signal) => { + signals.push(signal); + return new Promise(() => undefined); + }); + const ownerTicket = owner.request(input()); + owner.dispose(); + owner.dispose(); + await expect(ownerTicket.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + expect(signals[0]?.aborted).toBe(true); + await expect(owner.request(input()).result).resolves.toMatchObject({ + reason: "disposed", + }); + + const next = coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + }); + if (next.status !== "prepared") throw new Error("Expected owner"); + const coordinatorTicket = next.owner.request(input()); + coordinator.dispose(); + coordinator.dispose(); + await expect(coordinatorTicket.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + expect(coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + })).toEqual({ reason: "disposed", status: "unavailable" }); + }); + + it.each([ + { + expected: "provider-failed", + load: () => { + throw new Error("sync"); + }, + }, + { + expected: "provider-failed", + load: () => Promise.reject(new Error("async")), + }, + { + expected: "malformed-response", + load: () => ({ epoch, relations: [] }), + }, + ])("contains provider failure as $expected", async ({ expected, load }) => { + const { owner } = setup(load); + await expect(owner.request(input()).result).resolves.toEqual({ + reason: expected, + status: "unavailable", + }); + }); + + it("settles a revoked-array provider response as malformed", async () => { + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + const { owner } = setup(() => ({ + epoch, + relations: revoked.proxy, + })); + await expect(owner.request(input()).result).resolves.toEqual({ + reason: "malformed-response", + status: "unavailable", + }); + }); + + it("contains an unexpected decoder exception", async () => { + const integerCheck = vi.spyOn(Number, "isSafeInteger"); + const { owner } = setup((request) => { + integerCheck.mockImplementationOnce(() => { + throw new Error("decoder"); + }); + return ready(request); + }); + await expect(owner.request(input()).result).resolves.toEqual({ + reason: "malformed-response", + status: "unavailable", + }); + integerCheck.mockRestore(); + }); + + it("bounds cache entries with deterministic LRU eviction", async () => { + const calls = new Map(); + const { owner } = setup((request) => { + const key = request.relations[0]?.path[0]?.value ?? ""; + calls.set(key, (calls.get(key) ?? 0) + 1); + return ready(request); + }, 2); + await owner.request(input([reference("a")])).result; + await owner.request(input([reference("b")])).result; + await owner.request(input([reference("a")])).result; + await owner.request(input([reference("c")])).result; + await owner.request(input([reference("b")])).result; + + expect(calls).toEqual(new Map([ + ["a", 1], + ["b", 2], + ["c", 1], + ])); + }); + + it("rejects invalid providers, options, owners, and requests", async () => { + expect(createSqlColumnCatalogBatchCoordinator(null)).toEqual({ + reason: "invalid-provider", + status: "unavailable", + }); + expect(createSqlColumnCatalogBatchCoordinator({ + provider: {}, + })).toEqual({ + reason: "invalid-provider", + status: "unavailable", + }); + expect(createSqlColumnCatalogBatchCoordinator({ + maxCacheEntries: 0, + provider: { id: "catalog", loadColumns: vi.fn() }, + })).toEqual({ + reason: "invalid-options", + status: "unavailable", + }); + const created = createSqlColumnCatalogBatchCoordinator({ + provider: { id: "catalog", loadColumns: vi.fn() }, + }); + if (created.status !== "created") throw new Error("Expected coordinator"); + expect(created.coordinator.prepareOwner({ + dialectId: "", + scope: "scope", + })).toMatchObject({ status: "unavailable" }); + const prepared = created.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + }); + if (prepared.status !== "prepared") throw new Error("Expected owner"); + const invalidTicket = prepared.owner.request({ + expectedEpoch: epoch, + relations: [], + searchPaths: [], + }); + invalidTicket.cancel(); + await expect(invalidTicket.result).resolves.toEqual({ + reason: "invalid-request", + status: "unavailable", + }); + }); + + it("contains hostile and missing request properties", async () => { + const { owner } = setup(vi.fn()); + const hostile = new Proxy({}, { + getOwnPropertyDescriptor() { + throw new Error("hostile descriptor"); + }, + }); + + const hostileTicket = Reflect.apply( + owner.request, + undefined, + [hostile], + ); + await expect(hostileTicket.result).resolves.toEqual({ + reason: "invalid-request", + status: "unavailable", + }); + const missingTicket = Reflect.apply( + owner.request, + undefined, + [{ + expectedEpoch: epoch, + relations: [reference("users")], + }], + ); + await expect(missingTicket.result).resolves.toEqual({ + reason: "invalid-request", + status: "unavailable", + }); + }); +}); diff --git a/src/vnext/__tests__/column-catalog-boundary.test.ts b/src/vnext/__tests__/column-catalog-boundary.test.ts new file mode 100644 index 0000000..2ab1f9e --- /dev/null +++ b/src/vnext/__tests__/column-catalog-boundary.test.ts @@ -0,0 +1,884 @@ +import { describe, expect, it, vi } from "vitest"; +import { + captureSqlColumnCatalogProvider, + createSqlColumnCatalogBatchRequest, + decodeSqlColumnCatalogBatchResponse, + MAX_COLUMN_BATCH_RELATIONS, + MAX_COLUMN_ENTITY_ID_LENGTH, + MAX_COLUMNS_PER_RELATION, + resolveSqlColumnCatalogProvider, +} from "../column-catalog-boundary.js"; + +const epoch = Object.freeze({ generation: 7, token: "epoch-7" }); +const path = Object.freeze([ + Object.freeze({ quoted: false, value: "users" }), +]); + +function request() { + const result = createSqlColumnCatalogBatchRequest({ + dialectId: "duckdb", + expectedEpoch: epoch, + relations: [ + { path, requestKey: "users" }, + { + path: [{ quoted: false, value: "events" }], + requestKey: "events", + }, + ], + scope: "connection-1", + searchPaths: [[{ quoted: false, value: "main" }]], + }); + if (result.status !== "accepted") { + throw new Error("Expected valid request"); + } + return result.value; +} + +function provider(loadColumns: (...arguments_: unknown[]) => unknown) { + const captured = captureSqlColumnCatalogProvider({ + id: "catalog", + loadColumns, + }); + if (captured.status !== "accepted") { + throw new Error("Expected valid provider"); + } + return captured.value; +} + +function readyResponse() { + return { + epoch, + relations: [ + { + columns: [ + { + columnEntityId: "column-name", + dataType: "VARCHAR", + identifier: { quoted: false, value: "name" }, + insertText: "name", + ordinal: 1, + }, + { + columnEntityId: "column-id", + detail: "primary key", + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: 0, + }, + ], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + { + code: "unavailable", + requestKey: "events", + retry: "next-request", + status: "failed", + }, + ], + }; +} + +describe("column catalog provider boundary", () => { + it("captures methods without retaining provider receivers", () => { + const calls: unknown[][] = []; + const captured = provider(function ( + this: unknown, + ...arguments_: unknown[] + ) { + calls.push([this, ...arguments_]); + return null; + }); + const resolved = resolveSqlColumnCatalogProvider(captured); + const signal = new AbortController().signal; + resolved?.loadColumns(request(), signal); + + expect(resolved?.id).toBe("catalog"); + expect(calls).toEqual([[undefined, request(), signal]]); + expect(Object.isFrozen(resolved)).toBe(true); + expect(resolveSqlColumnCatalogProvider({})).toBeNull(); + }); + + it("rejects inherited, accessor, and oversized provider data", () => { + let calls = 0; + const accessor = Object.defineProperty( + { id: "catalog" }, + "loadColumns", + { + get: () => { + calls += 1; + return () => undefined; + }, + }, + ); + expect(captureSqlColumnCatalogProvider(accessor)).toMatchObject({ + reason: "invalid-shape", + status: "malformed", + }); + expect(calls).toBe(0); + expect(captureSqlColumnCatalogProvider( + Object.create({ id: "catalog", loadColumns: () => undefined }), + ).status).toBe("malformed"); + expect(captureSqlColumnCatalogProvider({ + id: "x".repeat(257), + loadColumns: () => undefined, + }).status).toBe("malformed"); + expect(captureSqlColumnCatalogProvider(null).status).toBe("malformed"); + expect(resolveSqlColumnCatalogProvider(null)).toBeNull(); + expect(resolveSqlColumnCatalogProvider(1)).toBeNull(); + }); + + it("contains hostile own-key and descriptor traps", () => { + const ownKeys = new Proxy( + { id: "catalog", loadColumns: () => undefined }, + { + ownKeys() { + throw new Error("private ownKeys trap"); + }, + }, + ); + const descriptor = new Proxy( + { id: "catalog", loadColumns: () => undefined }, + { + getOwnPropertyDescriptor() { + throw new Error("private descriptor trap"); + }, + }, + ); + expect(captureSqlColumnCatalogProvider(ownKeys).status).toBe("malformed"); + expect(captureSqlColumnCatalogProvider(descriptor).status).toBe( + "malformed", + ); + }); +}); + +describe("column catalog batch request boundary", () => { + it("normalizes paths, search paths, and deterministic request order", () => { + const result = createSqlColumnCatalogBatchRequest({ + dialectId: "duckdb", + expectedEpoch: epoch, + relations: [ + { path, requestKey: "z" }, + { path, requestKey: "a" }, + ], + scope: "scope", + searchPaths: [[{ quoted: true, value: "Main" }]], + }); + expect(result).toMatchObject({ + status: "accepted", + value: { + relations: [ + { requestKey: "a" }, + { requestKey: "z" }, + ], + }, + }); + if (result.status === "accepted") { + expect(Object.isFrozen(result.value)).toBe(true); + expect(Object.isFrozen(result.value.relations)).toBe(true); + expect(Object.isFrozen(result.value.relations[0]?.path)).toBe(true); + expect(Object.isFrozen(result.value.searchPaths)).toBe(true); + } + }); + + it("rejects malformed, duplicate, sparse, and resource-heavy requests", () => { + const base = { + dialectId: "duckdb", + expectedEpoch: epoch, + scope: "scope", + searchPaths: [], + }; + for (const relations of [ + [], + [{ path: [], requestKey: "empty-path" }], + [{ path, requestKey: "" }], + [ + { path, requestKey: "same" }, + { path, requestKey: "same" }, + ], + Array.from( + { length: MAX_COLUMN_BATCH_RELATIONS + 1 }, + (_, index) => ({ path, requestKey: `r${index}` }), + ), + ]) { + expect(createSqlColumnCatalogBatchRequest({ + ...base, + relations, + }).status).toBe("malformed"); + } + const sparse: unknown[] = []; + sparse.length = 1; + expect(createSqlColumnCatalogBatchRequest({ + ...base, + relations: sparse, + }).status).toBe("malformed"); + expect(createSqlColumnCatalogBatchRequest({ + ...base, + relations: [{ + path, + relationEntityId: "x".repeat(MAX_COLUMN_ENTITY_ID_LENGTH + 1), + requestKey: "long", + }], + }).status).toBe("malformed"); + expect(createSqlColumnCatalogBatchRequest( + Object.create(base), + ).status).toBe("malformed"); + }); + + it("does not invoke request accessors or hostile array iterators", () => { + let getterCalls = 0; + const input = Object.defineProperty( + { + dialectId: "duckdb", + epoch, + relations: [{ path, requestKey: "users" }], + searchPaths: [], + }, + "scope", + { + get: () => { + getterCalls += 1; + return "scope"; + }, + }, + ); + expect(createSqlColumnCatalogBatchRequest(input).status).toBe( + "malformed", + ); + expect(getterCalls).toBe(0); + + const relations = [{ path, requestKey: "users" }]; + Object.defineProperty(relations, Symbol.iterator, { + value: () => { + throw new Error("iterator must not run"); + }, + }); + expect(createSqlColumnCatalogBatchRequest({ + dialectId: "duckdb", + expectedEpoch: epoch, + relations, + scope: "scope", + searchPaths: [], + }).status).toBe("accepted"); + }); + + it("rejects malformed epochs, path components, search paths, and array traps", () => { + const base = { + dialectId: "duckdb", + expectedEpoch: epoch, + relations: [{ path, requestKey: "users" }], + scope: "scope", + searchPaths: [], + }; + const badEpochs = [ + { generation: -1, token: "x" }, + { generation: 1.5, token: "x" }, + { generation: 1, token: "" }, + { generation: "1", token: "x" }, + ]; + for (const expectedEpoch of badEpochs) { + expect( + createSqlColumnCatalogBatchRequest({ ...base, expectedEpoch }).status, + ).toBe("malformed"); + } + + const sparsePath: unknown[] = []; + sparsePath.length = 1; + const sparseSearchPaths: unknown[] = []; + sparseSearchPaths.length = 1; + const malformedPaths = [ + sparsePath, + [null], + [{ quoted: "false", value: "users" }], + [{ quoted: false, value: "" }], + ]; + for (const malformedPath of malformedPaths) { + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + relations: [{ path: malformedPath, requestKey: "users" }], + }).status, + ).toBe("malformed"); + } + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + searchPaths: sparseSearchPaths, + }).status, + ).toBe("malformed"); + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + searchPaths: [["not-a-component"]], + }).status, + ).toBe("malformed"); + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + relations: [null], + }).status, + ).toBe("malformed"); + + const trappedLength = new Proxy( + [{ path, requestKey: "users" }], + { + getOwnPropertyDescriptor(target, key) { + if (key === "length") { + throw new Error("private length trap"); + } + return Reflect.getOwnPropertyDescriptor(target, key); + }, + }, + ); + const trappedElement = new Proxy( + [{ path, requestKey: "users" }], + { + getOwnPropertyDescriptor(target, key) { + if (key === "0") { + throw new Error("private element trap"); + } + return Reflect.getOwnPropertyDescriptor(target, key); + }, + }, + ); + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + relations: trappedLength, + }).status, + ).toBe("malformed"); + expect( + createSqlColumnCatalogBatchRequest({ + ...base, + relations: trappedElement, + }).status, + ).toBe("malformed"); + }); +}); + +describe("column catalog response boundary", () => { + it("decodes all coverage states with stable frozen provenance", () => { + const captured = provider(() => undefined); + const result = decodeSqlColumnCatalogBatchResponse( + captured, + request(), + readyResponse(), + ); + expect(result).toMatchObject({ + status: "accepted", + value: { + relations: [ + { + code: "unavailable", + requestKey: "events", + status: "failed", + }, + { + columns: [ + { columnEntityId: "column-id", ordinal: 0 }, + { columnEntityId: "column-name", ordinal: 1 }, + ], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + ], + }, + }); + if (result.status === "accepted") { + const users = result.value.relations[1]; + expect(Object.isFrozen(result.value)).toBe(true); + expect(Object.isFrozen(result.value.relations)).toBe(true); + if (users?.status === "ready") { + expect(users.columns[0]?.provenance).toEqual({ + columnEntityId: "column-id", + epoch, + providerId: "catalog", + relationEntityId: "relation-users", + scope: "connection-1", + }); + expect(Object.isFrozen(users.columns[0]?.provenance)).toBe(true); + } + } + }); + + it("accepts loading, partial, and identical duplicate columns", () => { + const captured = provider(() => undefined); + const request_ = request(); + const column = { + columnEntityId: "same", + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: 0, + }; + const result = decodeSqlColumnCatalogBatchResponse( + captured, + request_, + { + epoch, + relations: [ + { + columns: [column, { ...column }], + coverage: "partial", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + { requestKey: "events", status: "loading" }, + ], + }, + ); + expect(result).toMatchObject({ + status: "accepted", + value: { + relations: [ + { requestKey: "events", status: "loading" }, + { + columns: [{ columnEntityId: "same" }], + coverage: "partial", + requestKey: "users", + }, + ], + }, + }); + }); + + it.each([ + { + name: "wrong epoch", + value: { ...readyResponse(), epoch: { generation: 8, token: "x" } }, + }, + { + name: "missing relation", + value: { epoch, relations: [readyResponse().relations[0]] }, + }, + { + name: "unexpected relation", + value: { + epoch, + relations: [ + ...readyResponse().relations, + { requestKey: "other", status: "loading" }, + ], + }, + }, + { + name: "duplicate request key", + value: { + epoch, + relations: [ + readyResponse().relations[0], + readyResponse().relations[0], + ], + }, + }, + { + name: "conflicting duplicate column", + value: { + epoch, + relations: [ + { + ...readyResponse().relations[0], + columns: [ + { + columnEntityId: "x", + identifier: { quoted: false, value: "a" }, + insertText: "a", + ordinal: 0, + }, + { + columnEntityId: "x", + identifier: { quoted: false, value: "b" }, + insertText: "b", + ordinal: 0, + }, + ], + }, + readyResponse().relations[1], + ], + }, + }, + { + name: "too many columns", + value: { + epoch, + relations: [ + { + ...readyResponse().relations[0], + columns: Array.from( + { length: MAX_COLUMNS_PER_RELATION + 1 }, + (_, ordinal) => ({ + columnEntityId: `c${ordinal}`, + identifier: { + quoted: false, + value: `c${ordinal}`, + }, + insertText: `c${ordinal}`, + ordinal, + }), + ), + }, + readyResponse().relations[1], + ], + }, + }, + ])("rejects $name", ({ value }) => { + expect(decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + value, + ).status).toBe("malformed"); + }); + + it("contains response accessors, iterators, and invalid provider handles", () => { + let getterCalls = 0; + const response = Object.defineProperty( + { epoch }, + "relations", + { + get: () => { + getterCalls += 1; + return []; + }, + }, + ); + expect(decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + response, + ).status).toBe("malformed"); + expect(getterCalls).toBe(0); + + const value = readyResponse(); + Object.defineProperty(value.relations, Symbol.iterator, { + value: vi.fn(() => { + throw new Error("iterator must not run"); + }), + }); + expect(decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + value, + ).status).toBe("accepted"); + const invalid = Reflect.apply( + decodeSqlColumnCatalogBatchResponse, + undefined, + [{}, request(), value], + ); + expect(invalid.status).toBe("malformed"); + }); + + it("rejects revoked response arrays without throwing", () => { + const captured = provider(() => undefined); + for (const response of [ + (() => { + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + return { epoch, relations: revoked.proxy }; + })(), + (() => { + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + return { + epoch, + relations: [ + { + ...readyResponse().relations[0], + columns: revoked.proxy, + }, + readyResponse().relations[1], + ], + }; + })(), + ]) { + expect(() => + decodeSqlColumnCatalogBatchResponse( + captured, + request(), + response, + ) + ).not.toThrow(); + expect(decodeSqlColumnCatalogBatchResponse( + captured, + request(), + response, + )).toEqual({ + reason: "invalid-shape", + status: "malformed", + }); + } + }); + + it("rejects cross-variant relation fields", () => { + const captured = provider(() => undefined); + const invalid = [ + { + code: "unknown", + requestKey: "users", + retry: "never", + status: "loading", + }, + { + code: "unknown", + ...readyResponse().relations[0], + retry: "never", + }, + { + code: "unknown", + columns: [], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + retry: "never", + status: "failed", + }, + ]; + for (const relation of invalid) { + expect(decodeSqlColumnCatalogBatchResponse( + captured, + request(), + { + epoch, + relations: [relation, readyResponse().relations[1]], + }, + ).status).toBe("malformed"); + } + }); + + it("rejects NUL-delimited conflicting column identities", () => { + const result = decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + { + epoch, + relations: [ + { + columns: [ + { + columnEntityId: "same", + identifier: { quoted: false, value: "x\u0000y" }, + insertText: "z", + ordinal: 0, + }, + { + columnEntityId: "same", + identifier: { quoted: false, value: "x" }, + insertText: "y\u0000z", + ordinal: 0, + }, + ], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + readyResponse().relations[1], + ], + }, + ); + expect(result).toEqual({ + reason: "duplicate-column-entity-id", + status: "malformed", + }); + }); + + it("rejects malformed relation and column state combinations", () => { + const captured = provider(() => undefined); + const request_ = request(); + const validUsers = readyResponse().relations[0]; + const validEvents = readyResponse().relations[1]; + const invalidUsers = [ + null, + { requestKey: "", status: "loading" }, + { + columns: [], + coverage: "complete", + relationEntityId: "", + requestKey: "users", + status: "ready", + }, + { code: "mystery", requestKey: "users", retry: "never", status: "failed" }, + { + code: "unavailable", + requestKey: "users", + retry: "mystery", + status: "failed", + }, + { requestKey: "users", status: "mystery" }, + { + columns: [null], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + { + columns: [{ + columnEntityId: "", + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + { + columns: [{ + columnEntityId: "id", + identifier: null, + insertText: "id", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + { + columns: [{ + columnEntityId: "id", + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: -1, + }], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + ]; + for (const users of invalidUsers) { + expect( + decodeSqlColumnCatalogBatchResponse(captured, request_, { + epoch, + relations: [users, validEvents], + }).status, + ).toBe("malformed"); + } + + const sparseColumns: unknown[] = []; + sparseColumns.length = 1; + const sparseRelations: unknown[] = []; + sparseRelations.length = 2; + expect( + decodeSqlColumnCatalogBatchResponse(captured, request_, { + epoch, + relations: [ + { ...validUsers, columns: sparseColumns }, + validEvents, + ], + }).status, + ).toBe("malformed"); + expect( + decodeSqlColumnCatalogBatchResponse(captured, request_, { + epoch, + relations: sparseRelations, + }).status, + ).toBe("malformed"); + }); + + it("rejects an unexpected relation without conflating it with response size", () => { + const result = decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + { + epoch, + relations: [ + readyResponse().relations[0], + { requestKey: "other", status: "loading" }, + ], + }, + ); + expect(result).toEqual({ + reason: "unexpected-relation", + status: "malformed", + }); + }); + + it("sorts stable ties and enforces the aggregate batch column cap", () => { + const tied = { + epoch, + relations: [ + { + columns: [ + { + columnEntityId: "b", + identifier: { quoted: true, value: "same" }, + insertText: "same", + ordinal: 0, + }, + { + columnEntityId: "a", + identifier: { quoted: true, value: "same" }, + insertText: "same", + ordinal: 0, + }, + ], + coverage: "complete", + relationEntityId: "relation-users", + requestKey: "users", + status: "ready", + }, + readyResponse().relations[1], + ], + }; + const sorted = decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + request(), + tied, + ); + expect(sorted).toMatchObject({ + status: "accepted", + value: { + relations: [ + { requestKey: "events" }, + { + columns: [ + { columnEntityId: "a" }, + { columnEntityId: "b" }, + ], + }, + ], + }, + }); + + const references = Array.from({ length: 17 }, (_, index) => ({ + path, + requestKey: `r${index}`, + })); + const created = createSqlColumnCatalogBatchRequest({ + dialectId: "duckdb", + expectedEpoch: epoch, + relations: references, + scope: "scope", + searchPaths: [], + }); + if (created.status !== "accepted") { + throw new Error("Expected aggregate-limit request"); + } + const relations = references.map((reference, relationIndex) => ({ + columns: Array.from({ length: MAX_COLUMNS_PER_RELATION }, (_, ordinal) => ({ + columnEntityId: `c${relationIndex}-${ordinal}`, + identifier: { quoted: false, value: `c${ordinal}` }, + insertText: `c${ordinal}`, + ordinal, + })), + coverage: "complete", + relationEntityId: `relation-${relationIndex}`, + requestKey: reference.requestKey, + status: "ready", + })); + expect( + decodeSqlColumnCatalogBatchResponse( + provider(() => undefined), + created.value, + { epoch, relations }, + ), + ).toEqual({ reason: "resource-limit", status: "malformed" }); + }); +}); diff --git a/src/vnext/__tests__/column-completion.test.ts b/src/vnext/__tests__/column-completion.test.ts new file mode 100644 index 0000000..f86655c --- /dev/null +++ b/src/vnext/__tests__/column-completion.test.ts @@ -0,0 +1,559 @@ +import { describe, expect, it } from "vitest"; +import { + composeSqlColumnCompletion, + prepareSqlColumnCatalogRelations, +} from "../column-completion.js"; +import { + MAX_COLUMN_BATCH_RELATIONS, +} from "../column-catalog-boundary.js"; +import type { + SqlColumnCatalogBatchOutcome, +} from "../column-catalog-batch-coordinator.js"; +import type { + SqlColumnCatalogResolvedColumn, +} from "../column-catalog-types.js"; +import type { + SqlColumnQuerySiteResult, +} from "../column-query-site.js"; +import { + POSTGRESQL_SQL_RELATION_DIALECT, +} from "../relation-dialect.js"; + +type ReadySite = Extract< + SqlColumnQuerySiteResult, + { status: "ready" } +>; + +const epoch = Object.freeze({ generation: 1, token: "epoch-1" }); + +function site( + qualifier: ReadySite["qualifier"] = [], + prefix = "", +): ReadySite { + return Object.freeze({ + coverage: "complete", + issues: Object.freeze([]), + prefix: Object.freeze({ quoted: false, value: prefix }), + qualifier: Object.freeze(qualifier), + relations: Object.freeze([ + Object.freeze({ + alias: Object.freeze({ quoted: false, value: "u" }), + path: Object.freeze([ + Object.freeze({ quoted: false, value: "app" }), + Object.freeze({ quoted: false, value: "users" }), + ]), + range: Object.freeze({ from: 20, to: 29 }), + }), + Object.freeze({ + alias: Object.freeze({ quoted: false, value: "o" }), + path: Object.freeze([ + Object.freeze({ quoted: false, value: "orders" }), + ]), + range: Object.freeze({ from: 35, to: 41 }), + }), + ]), + replacementRange: Object.freeze({ from: 9, to: 9 + prefix.length }), + status: "ready", + }); +} + +function column( + name: string, + relationEntityId: string, + ordinal: number, +): SqlColumnCatalogResolvedColumn { + return Object.freeze({ + columnEntityId: `${relationEntityId}:${name}`, + dataType: "VARCHAR", + identifier: Object.freeze({ quoted: false, value: name }), + insertText: name, + ordinal, + provenance: Object.freeze({ + columnEntityId: `${relationEntityId}:${name}`, + epoch, + providerId: "columns", + relationEntityId, + scope: "engine:1", + }), + }); +} + +function usable( + relations: Extract< + SqlColumnCatalogBatchOutcome, + { status: "usable" } + >["relations"], +): Extract { + return Object.freeze({ + epoch, + providerId: "columns", + relations: Object.freeze(relations), + scope: "engine:1", + status: "usable", + }); +} + +describe("column completion", () => { + it("bounds relation batches and reports the omitted bindings", () => { + const relations = Array.from( + { length: MAX_COLUMN_BATCH_RELATIONS + 1 }, + (_, index) => Object.freeze({ + alias: Object.freeze({ + quoted: false, + value: `t_${index}`, + }), + path: Object.freeze([ + Object.freeze({ + quoted: false, + value: `table_${index}`, + }), + ]), + range: Object.freeze({ from: index, to: index + 1 }), + }), + ); + const current = Object.freeze({ + ...site(), + relations: Object.freeze(relations), + }); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + + expect(prepared).toMatchObject({ + coverage: "partial", + references: { length: MAX_COLUMN_BATCH_RELATIONS }, + }); + expect(composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([]), + prepared, + providerId: "columns", + site: current, + })).toMatchObject({ + sources: [{ coverage: "partial", outcome: "ready" }], + value: { + isIncomplete: true, + issues: [{ reason: "query-binding-partial" }], + }, + }); + }); + + it("batches only the relation selected by an alias qualifier", () => { + const current = site([{ quoted: false, value: "u" }], "na"); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + + expect(prepared.references).toEqual([{ + path: [ + { quoted: false, value: "app" }, + { quoted: false, value: "users" }, + ], + requestKey: "binding:0", + }]); + }); + + it("matches an unaliased multipart path suffix", () => { + const current = Object.freeze({ + ...site([{ quoted: false, value: "app" }, { + quoted: false, + value: "users", + }]), + relations: Object.freeze([ + Object.freeze({ + alias: null, + path: Object.freeze([ + Object.freeze({ quoted: false, value: "app" }), + Object.freeze({ quoted: false, value: "users" }), + ]), + range: Object.freeze({ from: 20, to: 29 }), + }), + ]), + }); + expect( + prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ).references, + ).toHaveLength(1); + }); + + it("composes deterministic exact edits and provenance", () => { + const current = site([], "na"); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const result = composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([ + { + columns: [ + column("nickname", "users", 2), + column("name", "users", 1), + ], + coverage: "complete", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }, + { + columns: [column("name", "orders", 1)], + coverage: "complete", + relationEntityId: "orders", + requestKey: "binding:1", + status: "ready", + }, + ]), + prepared, + providerId: "columns", + site: current, + }); + + expect(result?.value).toMatchObject({ + isIncomplete: false, + items: [ + { + detail: "VARCHAR — o", + edit: { from: 9, insert: "name", to: 11 }, + kind: "column", + label: "name", + provenance: { + columnEntityId: "orders:name", + kind: "column-catalog", + relationEntityId: "orders", + }, + }, + { + detail: "VARCHAR — u", + label: "name", + }, + ], + }); + }); + + it("reports partial, loading, and failed relation evidence", () => { + const current = Object.freeze({ + ...site(), + coverage: "partial" as const, + issues: Object.freeze(["derived-relation" as const]), + }); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const result = composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([ + { + columns: [ + column("id", "users", 0), + Object.freeze({ + ...column("id", "users", 1), + columnEntityId: "users:id-alternate", + insertText: "id_alternate", + provenance: Object.freeze({ + ...column("id", "users", 1).provenance, + columnEntityId: "users:id-alternate", + }), + }), + ], + coverage: "partial", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }, + { requestKey: "binding:1", status: "loading" }, + ]), + prepared, + providerId: "columns", + site: current, + }); + + expect(result?.value).toMatchObject({ + isIncomplete: true, + issues: [ + { reason: "query-binding-partial" }, + { reason: "column-catalog-partial" }, + { reason: "column-catalog-loading" }, + ], + }); + expect(result?.sources[0]).toMatchObject({ + feature: "column-catalog", + outcome: "loading", + }); + }); + + it("maps provider unavailability and suppresses cancellations", () => { + const current = site(); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: { + reason: "malformed-response", + status: "unavailable", + }, + prepared, + providerId: "columns", + site: current, + }), + ).toMatchObject({ + value: { + issues: [{ reason: "column-catalog-malformed" }], + }, + }); + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: { status: "cancelled" }, + prepared, + providerId: "columns", + site: current, + }), + ).toBeNull(); + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: { status: "superseded" }, + prepared, + providerId: "columns", + site: current, + }), + ).toBeNull(); + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: { + reason: "provider-failed", + status: "unavailable", + }, + prepared, + providerId: "columns", + site: current, + }), + ).toMatchObject({ + value: { + issues: [{ reason: "column-catalog-failed" }], + }, + }); + }); + + it("fails qualifier matching closed and labels unaliased relations", () => { + const current = Object.freeze({ + ...site([ + { quoted: false, value: "too" }, + { quoted: false, value: "deep" }, + { quoted: false, value: "path" }, + ]), + relations: Object.freeze([ + Object.freeze({ + alias: null, + path: Object.freeze([ + Object.freeze({ quoted: false, value: "app" }), + Object.freeze({ quoted: false, value: "users" }), + ]), + range: Object.freeze({ from: 20, to: 29 }), + }), + ]), + }); + expect( + prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ).references, + ).toEqual([]); + + const unqualified = Object.freeze({ + ...current, + qualifier: Object.freeze([]), + }); + const prepared = prepareSqlColumnCatalogRelations( + unqualified, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const noMetadata = Object.freeze({ + columnEntityId: "users:x", + identifier: Object.freeze({ quoted: false, value: "x" }), + insertText: "x", + ordinal: 0, + provenance: Object.freeze({ + columnEntityId: "users:x", + epoch, + providerId: "columns", + relationEntityId: "users", + scope: "engine:1", + }), + }); + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([{ + columns: [noMetadata], + coverage: "complete", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }]), + prepared, + providerId: "columns", + site: unqualified, + })?.value.items[0], + ).toMatchObject({ + detail: "app.users", + label: "x", + }); + }); + + it("contains malformed mappings, duplicates, prefix misses, and failures", () => { + const current = site([], "i"); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const repeated = column("id", "users", 0); + const result = composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([ + { + columns: [repeated, repeated, column("name", "users", 1)], + coverage: "complete", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }, + { + columns: [column("id", "missing", 0)], + coverage: "complete", + relationEntityId: "missing", + requestKey: "unknown", + status: "ready", + }, + { + code: "unavailable", + requestKey: "binding:1", + retry: "next-request", + status: "failed", + }, + ]), + prepared, + providerId: "columns", + site: current, + }); + + expect(result).toMatchObject({ + sources: [{ + coverage: "partial", + failures: [{ + code: "unavailable", + requestKey: "binding:1", + retry: "next-request", + }], + outcome: "ready", + }], + value: { + isIncomplete: true, + issues: [ + { reason: "column-catalog-malformed" }, + { reason: "column-catalog-failed" }, + ], + items: [{ label: "id" }], + }, + }); + + expect( + composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([{ + code: "unavailable", + requestKey: "binding:0", + retry: "next-request", + status: "failed", + }]), + prepared, + providerId: "columns", + site: current, + })?.sources[0], + ).toMatchObject({ + failures: [{ + code: "unavailable", + requestKey: "binding:0", + retry: "next-request", + }], + outcome: "failed", + }); + }); + + it("uses deterministic code-unit ordering", () => { + const current = site(); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const result = composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([{ + columns: [ + column("ä_value", "users", 0), + column("z_value", "users", 1), + ], + coverage: "complete", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }]), + prepared, + providerId: "columns", + site: current, + }); + + expect(result?.value.items.map((item) => item.label)).toEqual([ + "z_value", + "ä_value", + ]); + }); + + it("uses relation detail to order columns with equal labels", () => { + const current = site(); + const prepared = prepareSqlColumnCatalogRelations( + current, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + const result = composeSqlColumnCompletion({ + dialect: POSTGRESQL_SQL_RELATION_DIALECT, + outcome: usable([ + { + columns: [column("id", "users", 0)], + coverage: "complete", + relationEntityId: "users", + requestKey: "binding:0", + status: "ready", + }, + { + columns: [column("id", "orders", 0)], + coverage: "complete", + relationEntityId: "orders", + requestKey: "binding:1", + status: "ready", + }, + ]), + prepared, + providerId: "columns", + site: current, + }); + + expect(result?.value.items.map((item) => item.detail)).toEqual([ + "VARCHAR — o", + "VARCHAR — u", + ]); + expect(result?.value.items.map((item) => item.edit.insert)).toEqual([ + "id", + "id", + ]); + }); +}); diff --git a/src/vnext/__tests__/column-query-site.test.ts b/src/vnext/__tests__/column-query-site.test.ts new file mode 100644 index 0000000..abddfc7 --- /dev/null +++ b/src/vnext/__tests__/column-query-site.test.ts @@ -0,0 +1,486 @@ +import { describe, expect, it } from "vitest"; +import { MAX_BOUNDED_SQL_LEXEMES } from "../bounded-sql-lexer.js"; +import { + MAX_COLUMN_QUERY_RELATIONS, + recognizeSqlColumnQuerySite, + type SqlColumnQuerySiteResult, +} from "../column-query-site.js"; +import { + BIGQUERY_SQL_RELATION_DIALECT, + DUCKDB_SQL_RELATION_DIALECT, + POSTGRESQL_SQL_RELATION_DIALECT, + type SqlRelationDialectRuntime, +} from "../relation-dialect.js"; +import { + createIdentitySqlSource, + createMaskedSqlSource, +} from "../source.js"; +import { + buildSqlStatementIndex, + findSqlStatementSlot, +} from "../statement-index.js"; + +function analyze( + marked: string, + options: { + readonly dialect?: SqlRelationDialectRuntime; + readonly regions?: readonly { + readonly from: number; + readonly language: string; + readonly to: number; + }[]; + } = {}, +): SqlColumnQuerySiteResult { + const position = marked.indexOf("|"); + if (position < 0 || marked.indexOf("|", position + 1) >= 0) { + throw new Error("Fixture requires exactly one cursor marker"); + } + const text = marked.slice(0, position) + marked.slice(position + 1); + const dialect = options.dialect ?? POSTGRESQL_SQL_RELATION_DIALECT; + const source = options.regions + ? createMaskedSqlSource(text, options.regions) + : createIdentitySqlSource(text); + const index = buildSqlStatementIndex( + source.analysisText, + dialect.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot(index, position, "left"); + return recognizeSqlColumnQuerySite( + source, + slot, + position, + dialect, + ); +} + +function ready( + result: SqlColumnQuerySiteResult, +): Extract { + if (result.status !== "ready") { + throw new Error(`Expected ready, received ${JSON.stringify(result)}`); + } + expect(result.status).toBe("ready"); + return result; +} + +describe("recognizeSqlColumnQuerySite", () => { + it("finds a qualified SELECT-list site and its aliased relation", () => { + const result = ready( + analyze("SELECT u.na| FROM app.users AS u"), + ); + + expect(result).toMatchObject({ + coverage: "complete", + prefix: { quoted: false, value: "na" }, + qualifier: [{ quoted: false, value: "u" }], + replacementRange: { from: 9, to: 11 }, + }); + expect(result.relations).toEqual([{ + alias: { quoted: false, value: "u" }, + path: [ + { quoted: false, value: "app" }, + { quoted: false, value: "users" }, + ], + range: { from: 17, to: 26 }, + }]); + }); + + it("collects comma and join relations for expression clauses", () => { + const result = ready( + analyze( + "SELECT * FROM users u, ignored i JOIN orders AS o ON o.id = u.id WHERE cr|", + ), + ); + + expect(result.prefix.value).toBe("cr"); + expect(result.qualifier).toEqual([]); + expect(result.relations.map((relation) => relation.alias?.value)) + .toEqual(["u", "i", "o"]); + }); + + it("includes correlated outer-query relations", () => { + const result = ready( + analyze( + "SELECT * FROM outer_table o WHERE EXISTS (SELECT i.na| FROM inner_table i)", + ), + ); + + expect(result.qualifier[0]?.value).toBe("i"); + expect(result.coverage).toBe("partial"); + expect(result.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(["inner_table", "outer_table"]); + }); + + it("does not correlate an ordinary derived table", () => { + const result = ready( + analyze( + "SELECT * FROM (SELECT i.na| FROM inner_table i) d JOIN outer_table o ON true", + ), + ); + + expect(result.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(["inner_table"]); + }); + + it("correlates LATERAL tables only to preceding relations", () => { + for ( + const dialect of [ + POSTGRESQL_SQL_RELATION_DIALECT, + DUCKDB_SQL_RELATION_DIALECT, + ] + ) { + for (const expression of ["u.na|", "na|"]) { + const result = ready( + analyze( + `SELECT * FROM users u, LATERAL (SELECT ${expression} FROM orders o) x, secrets s`, + { dialect }, + ), + ); + + expect(result.relations.map((relation) => + relation.alias?.value + )).toEqual(["o", "u"]); + } + } + }); + + it("limits correlation in a JOIN condition to prior relations", () => { + const result = ready( + analyze( + "SELECT * FROM users u JOIN orders o ON EXISTS (SELECT u.|) JOIN secrets s ON true", + ), + ); + + expect(result.relations.map((relation) => + relation.alias?.value + )).toEqual(["u", "o"]); + }); + + it("walks nested correlation without adopting a closed sibling", () => { + const nested = ready( + analyze( + "SELECT * FROM root_table r WHERE EXISTS (SELECT * FROM mid_table m WHERE EXISTS (SELECT r.|))", + ), + ); + expect(nested.relations.map((relation) => + relation.alias?.value + )).toEqual(["m", "r"]); + + const sibling = ready( + analyze( + "SELECT * FROM root_table r WHERE EXISTS (SELECT 1) AND EXISTS (SELECT r.|)", + ), + ); + expect(sibling.relations.map((relation) => + relation.alias?.value + )).toEqual(["r"]); + }); + + it("allows a parenthesized top-level query without an outer scope", () => { + const result = ready(analyze("(SELECT | FROM inner_table)")); + expect(result.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(["inner_table"]); + }); + + it("keeps set-operation arms and JOIN visibility isolated", () => { + const firstArm = ready( + analyze("SELECT | FROM users UNION SELECT x FROM secrets"), + ); + expect(firstArm.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(["users"]); + + const secondArm = ready( + analyze("SELECT x FROM users UNION SELECT | FROM secrets"), + ); + expect(secondArm.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(["secrets"]); + + const joinCondition = ready( + analyze( + "SELECT * FROM users u JOIN orders o ON o.user_id = u.| JOIN payments p ON true", + ), + ); + expect(joinCondition.relations.map((relation) => + relation.alias?.value + )).toEqual(["u", "o"]); + }); + + it("supports BigQuery quoted multipart bindings", () => { + const result = ready( + analyze("SELECT t.| FROM `project.dataset.table` AS t", { + dialect: BIGQUERY_SQL_RELATION_DIALECT, + }), + ); + + expect(result.relations[0]).toMatchObject({ + alias: { quoted: false, value: "t" }, + path: [ + { quoted: true, value: "project" }, + { quoted: true, value: "dataset" }, + { quoted: true, value: "table" }, + ], + }); + }); + + it.each([ + ["SELECT * FROM us|", "not-column-position"], + ["INSERT INTO users VALUES (|)", "not-select-query"], + ["SELECT 'abc|' FROM users", "cursor-in-string"], + ["SELECT /* abc| */ 1 FROM users", "cursor-in-comment"], + ] as const)("fails closed for %s", (marked, reason) => { + expect(analyze(marked)).toMatchObject({ reason, status: "inactive" }); + }); + + it("fails closed inside an embedded region", () => { + const marked = "SELECT {value|} FROM users"; + const position = marked.indexOf("|"); + expect( + analyze(marked, { + regions: [{ + from: marked.indexOf("{"), + language: "python", + to: marked.indexOf("}") + 1, + }], + }), + ).toMatchObject({ + reason: "cursor-in-embedded-region", + status: "inactive", + }); + expect(position).toBeGreaterThan(0); + }); + + it("marks derived and table-function evidence partial", () => { + expect( + ready( + analyze( + "SELECT x.| FROM (SELECT 1) x JOIN UNNEST(items) AS item ON true", + ), + ), + ).toMatchObject({ + coverage: "partial", + issues: [ + "derived-relation", + "nested-query", + "table-function", + ], + }); + }); + + it.each([ + "SELECT * FROM users WHERE na|", + "SELECT * FROM users GROUP BY na|", + "SELECT * FROM users HAVING na|", + "SELECT * FROM users QUALIFY na|", + "SELECT * FROM users ORDER BY na|", + "SELECT * FROM users u JOIN orders o ON o.id = u.|", + "SELECT * FROM users u JOIN orders o USING (i|)", + ])("recognizes expression clause sites in %s", (marked) => { + expect(ready(analyze(marked)).relations.length).toBeGreaterThan(0); + }); + + it.each([ + "SELECT * FROM users LIMIT |", + "SELECT * FROM users FETCH |", + "SELECT * FROM users OFFSET |", + ])("rejects row-limit clause sites in %s", (marked) => { + expect(analyze(marked)).toMatchObject({ + reason: "not-column-position", + status: "inactive", + }); + }); + + it("supports implicit quoted aliases and incomplete relations", () => { + expect( + ready(analyze('SELECT "u".| FROM app.users "u"')).relations[0], + ).toMatchObject({ + alias: { quoted: true, value: "u" }, + }); + expect(ready(analyze("SELECT | FROM"))).toMatchObject({ + coverage: "partial", + issues: ["incomplete-relation"], + relations: [], + }); + expect(ready(analyze('SELECT | FROM ""'))).toMatchObject({ + coverage: "partial", + issues: ["incomplete-relation"], + relations: [], + }); + expect(ready(analyze("SELECT | FROM users AS"))).toMatchObject({ + coverage: "partial", + issues: ["incomplete-relation"], + relations: [{ + alias: null, + path: [{ value: "users" }], + }], + }); + expect(ready(analyze("SELECT | FROM users AS WHERE true"))) + .toMatchObject({ + coverage: "partial", + issues: ["incomplete-relation"], + }); + expect(ready(analyze('SELECT | FROM users AS "u"')).relations[0]) + .toMatchObject({ + alias: { quoted: true, value: "u" }, + }); + }); + + it("fails closed for malformed typed paths and line comments", () => { + expect(analyze("SELECT u..| FROM users u")).toMatchObject({ + reason: "not-column-position", + status: "inactive", + }); + expect(analyze("SELECT 1 -- co|mment\nFROM users")).toMatchObject({ + reason: "cursor-in-comment", + status: "inactive", + }); + expect(analyze("SELECT 1 -- comment|")).toMatchObject({ + reason: "cursor-in-comment", + status: "inactive", + }); + expect(analyze("SELECT 1 -- comment|\nFROM users")).toMatchObject({ + reason: "cursor-in-comment", + status: "inactive", + }); + expect( + analyze("SELECT 1 # comment|", { + dialect: BIGQUERY_SQL_RELATION_DIALECT, + }), + ).toMatchObject({ + reason: "cursor-in-comment", + status: "inactive", + }); + }); + + it("bounds lexical work before relation analysis", () => { + const expression = Array.from( + { length: MAX_BOUNDED_SQL_LEXEMES + 1 }, + () => "x", + ).join("+"); + expect(analyze(`SELECT ${expression}| FROM users`)).toMatchObject({ + reason: "resource-limit", + status: "unavailable", + }); + }); + + it("bounds relation materialization", () => { + const joins = Array.from( + { length: MAX_COLUMN_QUERY_RELATIONS + 1 }, + (_, index) => ` JOIN table_${index} t_${index} ON true`, + ).join(""); + expect(analyze(`SELECT | FROM base b${joins}`)).toMatchObject({ + reason: "resource-limit", + status: "unavailable", + }); + expect(analyze(`SELECT (SELECT |) FROM base b${joins}`)) + .toMatchObject({ + reason: "resource-limit", + status: "unavailable", + }); + }); + + it("rejects foreign contract values without inspecting them", () => { + let inspected = false; + const hostile = new Proxy({}, { + get() { + inspected = true; + throw new Error("hostile"); + }, + }); + expect( + Reflect.apply(recognizeSqlColumnQuerySite, undefined, [ + hostile, + hostile, + 0, + hostile, + ]), + ).toMatchObject({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + expect(inspected).toBe(false); + }); + + it("rejects each unauthenticated argument and invalid position", () => { + const source = createIdentitySqlSource("SELECT |"); + const position = source.analysisText.indexOf("|"); + const index = buildSqlStatementIndex( + source.analysisText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot(index, position, "left"); + for (const input of [ + [{}, slot, position, POSTGRESQL_SQL_RELATION_DIALECT], + [source, {}, position, POSTGRESQL_SQL_RELATION_DIALECT], + [source, slot, position, {}], + [source, slot, Number.NaN, POSTGRESQL_SQL_RELATION_DIALECT], + [source, slot, -1, POSTGRESQL_SQL_RELATION_DIALECT], + [ + source, + slot, + source.analysisText.length + 1, + POSTGRESQL_SQL_RELATION_DIALECT, + ], + ]) { + expect( + Reflect.apply(recognizeSqlColumnQuerySite, undefined, input), + ).toMatchObject({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + } + }); + + it("rejects a cursor outside its authenticated exact slot", () => { + const text = + "SELECT x FROM first_table WHERE ; SELECT y FROM second_table"; + const source = createIdentitySqlSource(text); + const index = buildSqlStatementIndex( + text, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const first = findSqlStatementSlot(index, 0, "right"); + const secondPosition = text.indexOf("y FROM"); + + expect( + recognizeSqlColumnQuerySite( + source, + first, + secondPosition, + POSTGRESQL_SQL_RELATION_DIALECT, + ), + ).toEqual({ + reason: "not-column-position", + status: "inactive", + }); + }); + + it("preserves opaque statement failures", () => { + const source = createIdentitySqlSource("DELIMITER $$"); + const index = buildSqlStatementIndex( + source.analysisText, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const slot = findSqlStatementSlot( + index, + source.analysisText.length, + "left", + ); + expect(slot.boundaryQuality).toBe("opaque"); + expect( + recognizeSqlColumnQuerySite( + source, + slot, + source.analysisText.length, + POSTGRESQL_SQL_RELATION_DIALECT, + ), + ).toMatchObject({ + reason: "opaque-statement", + status: "unavailable", + }); + }); +}); diff --git a/src/vnext/__tests__/hostile-data.test.ts b/src/vnext/__tests__/hostile-data.test.ts new file mode 100644 index 0000000..f5194d2 --- /dev/null +++ b/src/vnext/__tests__/hostile-data.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; +import { isDataArray } from "../types.js"; + +describe("hostile data arrays", () => { + it("recognizes arrays without invoking elements", () => { + expect(isDataArray([])).toBe(true); + expect(isDataArray({})).toBe(false); + }); + + it("contains revoked and hostile proxy traps", () => { + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + expect(isDataArray(revoked.proxy)).toBe(false); + }); +}); diff --git a/src/vnext/__tests__/local-relation-site.test.ts b/src/vnext/__tests__/local-relation-site.test.ts index 899e6c1..1a1c56f 100644 --- a/src/vnext/__tests__/local-relation-site.test.ts +++ b/src/vnext/__tests__/local-relation-site.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + analyzeSqlLocalColumnSite, analyzeSqlLocalRelationSite, prepareSqlLocalRelationStatement, type SqlLocalRelationSiteResult, @@ -411,6 +412,34 @@ describe("local relation-site evidence", () => { ); }); + it("fails column analysis closed outside the prepared statement", () => { + const text = + "SELECT x FROM first_table WHERE ; SELECT y FROM second_table"; + const source = createIdentitySqlSource(text); + const index = buildSqlStatementIndex( + text, + POSTGRESQL_SQL_RELATION_DIALECT.querySite.lexicalProfile, + ); + const first = findSqlStatementSlot(index, 0, "right"); + const preparation = prepareSqlLocalRelationStatement( + source, + index, + first, + POSTGRESQL_SQL_RELATION_DIALECT, + ); + if (preparation.status !== "ready") { + throw new Error("Expected exact first-statement preparation"); + } + + expect(analyzeSqlLocalColumnSite( + preparation.statement, + text.indexOf("y FROM"), + )).toEqual({ + reason: "not-column-position", + status: "inactive", + }); + }); + it("separates qualified sites from irrelevant CTE uncertainty", () => { const ready = expectReady( analyzeMarked( @@ -495,6 +524,24 @@ describe("local relation-site evidence", () => { reason: "ambiguous-query-site", status: "unavailable", }); + expect( + Reflect.apply(analyzeSqlLocalColumnSite, undefined, [ + null, + fixture.position, + ]), + ).toEqual({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + expect( + analyzeSqlLocalColumnSite( + { ...preparation.statement }, + fixture.position, + ), + ).toEqual({ + reason: "ambiguous-query-site", + status: "unavailable", + }); }); it("is deterministic and freezes every exposed wrapper", () => { diff --git a/src/vnext/__tests__/namespace-catalog-boundary.test.ts b/src/vnext/__tests__/namespace-catalog-boundary.test.ts new file mode 100644 index 0000000..bf2cae6 --- /dev/null +++ b/src/vnext/__tests__/namespace-catalog-boundary.test.ts @@ -0,0 +1,417 @@ +import { describe, expect, it, vi } from "vitest"; +import { + captureSqlNamespaceCatalogProvider, + createSqlNamespaceCatalogSearchRequest, + decodeSqlNamespaceCatalogSearchResponse, + MAX_NAMESPACE_RESULTS, + resolveSqlNamespaceCatalogProvider, +} from "../namespace-catalog-boundary.js"; + +const epoch = Object.freeze({ generation: 1, token: "one" }); + +function request(expectedEpoch = epoch) { + const result = createSqlNamespaceCatalogSearchRequest({ + dialectId: "duckdb", + expectedEpoch, + limit: 10, + prefix: { quoted: false, value: "ma" }, + qualifier: [{ quoted: false, value: "memory" }], + scope: "connection", + searchPaths: [[{ quoted: false, value: "main" }]], + }); + if (result.status !== "accepted") throw new Error("request"); + return result.value; +} + +function captured(search: unknown = vi.fn()) { + const result = captureSqlNamespaceCatalogProvider({ + id: "namespaces", + search, + }); + if (result.status !== "accepted") throw new Error("provider"); + return result.value; +} + +function container( + id: string, + role: "catalog" | "dataset" | "project" | "schema", + value: string, +) { + return { + canonicalPath: [{ + quoted: false, + role, + value, + }], + containerEntityId: id, + detail: `${role} detail`, + insertText: value, + matchQuality: "exact", + }; +} + +describe("namespace catalog boundary", () => { + it("captures provider methods without exposing a receiver", () => { + const receivers: unknown[] = []; + const search = function (this: unknown) { + receivers.push(this); + }; + const handle = captured(search); + const context = resolveSqlNamespaceCatalogProvider(handle); + expect(context?.id).toBe("namespaces"); + context?.search(request(), new AbortController().signal); + expect(receivers).toEqual([undefined]); + expect(Reflect.apply( + resolveSqlNamespaceCatalogProvider, + undefined, + [{}], + )).toBeNull(); + expect(Object.isFrozen(handle)).toBe(true); + }); + + it("rejects hostile or invalid providers", () => { + const accessor = {}; + Object.defineProperty(accessor, "id", { get: vi.fn() }); + expect(captureSqlNamespaceCatalogProvider(accessor).status) + .toBe("malformed"); + expect(captureSqlNamespaceCatalogProvider(null).status) + .toBe("malformed"); + expect(captureSqlNamespaceCatalogProvider({ + id: "", + search() {}, + }).status).toBe("malformed"); + expect(captureSqlNamespaceCatalogProvider({ + id: "x", + search: 1, + }).status).toBe("malformed"); + expect(captureSqlNamespaceCatalogProvider({ + extra: true, + id: "x", + search() {}, + }).status).toBe("malformed"); + expect(captureSqlNamespaceCatalogProvider( + new Proxy({}, { ownKeys: () => { throw new Error("trap"); } }), + ).status).toBe("malformed"); + }); + + it("normalizes and freezes one bounded query-site search", () => { + const value = request(); + expect(value).toEqual({ + dialectId: "duckdb", + expectedEpoch: epoch, + limit: 10, + prefix: { quoted: false, value: "ma" }, + qualifier: [{ quoted: false, value: "memory" }], + scope: "connection", + searchPaths: [[{ quoted: false, value: "main" }]], + }); + expect(Object.isFrozen(value)).toBe(true); + expect(Object.isFrozen(value.prefix)).toBe(true); + expect(Object.isFrozen(value.qualifier)).toBe(true); + expect(Object.isFrozen(value.searchPaths[0])).toBe(true); + expect(createSqlNamespaceCatalogSearchRequest({ + ...value, + expectedEpoch: null, + }).status).toBe("accepted"); + }); + + it("rejects malformed, sparse, accessor, and oversized requests", () => { + const valid = request(); + const sparse: unknown[] = []; + sparse.length = 1; + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + const cases: unknown[] = [ + null, + { ...valid, extra: true }, + { ...valid, dialectId: "" }, + { ...valid, scope: "" }, + { ...valid, expectedEpoch: undefined }, + { ...valid, expectedEpoch: { generation: -1, token: "x" } }, + { ...valid, expectedEpoch: { generation: 1.5, token: "x" } }, + { ...valid, expectedEpoch: { generation: 1, token: "" } }, + { ...valid, limit: 0 }, + { ...valid, limit: MAX_NAMESPACE_RESULTS + 1 }, + { ...valid, limit: 1.5 }, + { ...valid, prefix: { quoted: "no", value: "x" } }, + { ...valid, prefix: { quoted: false, value: "x", extra: 1 } }, + { ...valid, qualifier: [{ quoted: false, value: "" }] }, + { ...valid, searchPaths: [[{ quoted: false, value: "" }]] }, + { ...valid, qualifier: sparse }, + { ...valid, searchPaths: sparse }, + { ...valid, qualifier: revoked.proxy }, + { ...valid, qualifier: [{ quoted: false, value: "x".repeat(257) }] }, + { ...valid, searchPaths: Array.from({ length: 33 }, () => []) }, + ]; + const accessor = { ...valid }; + Object.defineProperty(accessor, "prefix", { get: vi.fn() }); + cases.push(accessor); + cases.push({ + ...valid, + qualifier: new Proxy([], { + getOwnPropertyDescriptor: (_target, key) => { + if (key === "length") throw new Error("length trap"); + return undefined; + }, + }), + }); + cases.push({ + ...valid, + qualifier: new Proxy([{ quoted: false, value: "x" }], { + getOwnPropertyDescriptor: (target, key) => { + if (key === "0") throw new Error("element trap"); + return Reflect.getOwnPropertyDescriptor(target, key); + }, + }), + }); + for (const value of cases) { + expect(createSqlNamespaceCatalogSearchRequest(value).status) + .toBe("malformed"); + } + expect(createSqlNamespaceCatalogSearchRequest({ + ...valid, + prefix: { quoted: false, value: "" }, + qualifier: [], + }).status).toBe("accepted"); + }); + + it("decodes, deduplicates, orders, and freezes all container roles", () => { + const provider = captured(); + const decoded = decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + { + containers: [ + container("schema", "schema", "main"), + { ...container("schema", "schema", "main") }, + container("project", "project", "alpha"), + container("dataset", "dataset", "events"), + container("catalog", "catalog", "memory"), + ], + coverage: "partial", + epoch, + status: "ready", + }, + ); + expect(decoded.status).toBe("accepted"); + if (decoded.status !== "accepted" || + decoded.value.status !== "ready") { + throw new Error("Expected a ready namespace response"); + } + expect(decoded.value.containers.map((value) => + value.canonicalPath[0]?.role + )).toEqual(["catalog", "dataset", "project", "schema"]); + expect(decoded.value.containers[0]?.provenance).toEqual({ + containerEntityId: "catalog", + epoch, + providerId: "namespaces", + scope: "connection", + }); + expect(Object.isFrozen(decoded.value)).toBe(true); + expect(Object.isFrozen(decoded.value.containers)).toBe(true); + expect(Object.isFrozen( + decoded.value.containers[0]?.canonicalPath, + )).toBe(true); + }); + + it("decodes loading and normalized failures", () => { + const provider = captured(); + expect(decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + { epoch, status: "loading" }, + )).toMatchObject({ + status: "accepted", + value: { status: "loading" }, + }); + expect(decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + { + code: "rate-limited", + epoch, + retry: "next-request", + status: "failed", + }, + )).toMatchObject({ + status: "accepted", + value: { + code: "rate-limited", + retry: "next-request", + status: "failed", + }, + }); + }); + + it("orders tied identities and accepts quoted paths without detail", () => { + const provider = captured(); + const base = { + canonicalPath: [{ + quoted: true, + role: "schema", + value: "Main", + }], + insertText: "\"Main\"", + matchQuality: "exact", + }; + const decoded = decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + { + containers: [ + { ...base, containerEntityId: "z" }, + { ...base, containerEntityId: "a" }, + ], + coverage: "complete", + epoch, + status: "ready", + }, + ); + expect(decoded).toMatchObject({ + status: "accepted", + value: { + containers: [ + { containerEntityId: "a" }, + { containerEntityId: "z" }, + ], + }, + }); + }); + + it("orders raw identifier values by UTF-16 code units", () => { + const decoded = decodeSqlNamespaceCatalogSearchResponse( + captured(), + request(), + { + containers: [ + container("hash", "schema", "#"), + container("quote", "schema", "\""), + container("slash", "schema", "\\"), + container("control", "schema", "\u0000"), + ], + coverage: "complete", + epoch, + status: "ready", + }, + ); + expect(decoded).toMatchObject({ + status: "accepted", + value: { + containers: [ + { containerEntityId: "control" }, + { containerEntityId: "quote" }, + { containerEntityId: "hash" }, + { containerEntityId: "slash" }, + ], + }, + }); + }); + + it("compares canonical paths structurally when they contain NUL", () => { + const provider = captured(); + const base = { + containerEntityId: "same", + insertText: "value", + matchQuality: "exact", + }; + const decoded = decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + { + containers: [ + { + ...base, + canonicalPath: [{ + quoted: false, + role: "schema", + value: "a\u0000catalog:u:b", + }], + }, + { + ...base, + canonicalPath: [ + { quoted: false, role: "schema", value: "a" }, + { quoted: false, role: "catalog", value: "b" }, + ], + }, + ], + coverage: "complete", + epoch, + status: "ready", + }, + ); + expect(decoded).toEqual({ + reason: "duplicate-entity-id", + status: "malformed", + }); + }); + + it("rejects malformed, conflicting, stale, and oversized responses", () => { + const provider = captured(); + const valid = { + containers: [container("one", "schema", "main")], + coverage: "complete", + epoch, + status: "ready", + }; + const conflicting = { + ...valid, + containers: [ + container("same", "schema", "main"), + container("same", "schema", "other"), + ], + }; + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + const malformedContainers = [ + { ...container("x", "schema", "x"), canonicalPath: [] }, + { ...container("x", "schema", "x"), canonicalPath: [{ + quoted: false, + role: "schema", + value: "", + }] }, + { ...container("x", "schema", "x"), canonicalPath: [{ + quoted: false, + role: "table", + value: "x", + }] }, + { ...container("x", "schema", "x"), containerEntityId: "" }, + { ...container("x", "schema", "x"), insertText: "" }, + { ...container("x", "schema", "x"), matchQuality: "bad" }, + { ...container("x", "schema", "x"), detail: 1 }, + ]; + const cases: unknown[] = [ + null, + { ...valid, extra: true }, + { ...valid, epoch: { generation: 2, token: "two" } }, + { ...valid, coverage: "unknown" }, + { ...valid, containers: Array.from( + { length: 11 }, + (_, index) => container(String(index), "schema", String(index)), + ) }, + conflicting, + { ...valid, containers: [null] }, + { ...valid, containers: revoked.proxy }, + { epoch, extra: true, status: "loading" }, + { code: "bad", epoch, retry: "never", status: "failed" }, + { code: "unknown", epoch, retry: "bad", status: "failed" }, + { code: "unknown", epoch, retry: "never", status: "failed", x: 1 }, + ...malformedContainers.map((value) => ({ + ...valid, + containers: [value], + })), + ]; + for (const value of cases) { + expect(decodeSqlNamespaceCatalogSearchResponse( + provider, + request(), + value, + ).status).toBe("malformed"); + } + expect(Reflect.apply( + decodeSqlNamespaceCatalogSearchResponse, + undefined, + [{}, request(), valid], + )).toMatchObject({ status: "malformed" }); + }); +}); diff --git a/src/vnext/__tests__/namespace-catalog-coordinator.bench.ts b/src/vnext/__tests__/namespace-catalog-coordinator.bench.ts new file mode 100644 index 0000000..47489ee --- /dev/null +++ b/src/vnext/__tests__/namespace-catalog-coordinator.bench.ts @@ -0,0 +1,71 @@ +import { beforeAll, bench, describe } from "vitest"; +import { + createSqlNamespaceCatalogCoordinator, +} from "../namespace-catalog-coordinator.js"; +import type { + SqlNamespaceCatalogSearchRequest, +} from "../namespace-catalog-types.js"; + +const epoch = Object.freeze({ generation: 1, token: "bench" }); + +function response(request: SqlNamespaceCatalogSearchRequest) { + return { + containers: Array.from({ length: 100 }, (_, index) => ({ + canonicalPath: [ + { quoted: false, role: "catalog", value: "memory" }, + { quoted: false, role: "schema", value: `schema_${index}` }, + ], + containerEntityId: `schema:${index}`, + insertText: `schema_${index}`, + matchQuality: "exact", + })), + coverage: "complete", + epoch: request.expectedEpoch ?? epoch, + status: "ready", + }; +} + +function owner() { + const created = createSqlNamespaceCatalogCoordinator({ + provider: { + id: "benchmark", + search: (request: SqlNamespaceCatalogSearchRequest) => + Promise.resolve(response(request)), + }, + }); + if (created.status !== "created") throw new Error("coordinator"); + const prepared = created.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "benchmark", + }); + if (prepared.status !== "prepared") throw new Error("owner"); + return prepared.owner; +} + +const search = Object.freeze({ + expectedEpoch: epoch, + limit: 100, + prefix: Object.freeze({ quoted: false, value: "schema" }), + qualifier: Object.freeze([ + Object.freeze({ quoted: false, value: "memory" }), + ]), + searchPaths: Object.freeze([]), +}); + +describe("namespace catalog coordinator", () => { + bench("cold 100-container search", async () => { + const current = owner(); + await current.request(search).result; + current.dispose(); + }); + + const warm = owner(); + const primed = warm.request(search).result; + beforeAll(async () => { + await primed; + }); + + bench("warm 100-container cache lookup", async () => { + await warm.request(search).result; + }); +}); diff --git a/src/vnext/__tests__/namespace-catalog-coordinator.test.ts b/src/vnext/__tests__/namespace-catalog-coordinator.test.ts new file mode 100644 index 0000000..3760d80 --- /dev/null +++ b/src/vnext/__tests__/namespace-catalog-coordinator.test.ts @@ -0,0 +1,631 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createSqlNamespaceCatalogCoordinator, + type SqlNamespaceCatalogSearchOutcome, +} from "../namespace-catalog-coordinator.js"; +import { + composeSqlNamespaceCompletion, + prepareSqlNamespaceCatalogSearch, +} from "../namespace-completion.js"; +import type { + SqlNamespaceCatalogResolvedContainer, + SqlNamespaceCatalogSearchRequest, +} from "../namespace-catalog-types.js"; +import type { SqlCatalogEpoch } from "../relation-completion-types.js"; + +const epoch: SqlCatalogEpoch = + Object.freeze({ generation: 1, token: "one" }); +const nextEpoch: SqlCatalogEpoch = + Object.freeze({ generation: 2, token: "two" }); + +function input( + prefix = "ma", + expectedEpoch: SqlCatalogEpoch | null = epoch, +) { + return { + expectedEpoch, + limit: 10, + prefix: { quoted: false, value: prefix }, + qualifier: [{ quoted: false, value: "memory" }], + searchPaths: [[{ quoted: false, value: "main" }]], + }; +} + +function response( + request: SqlNamespaceCatalogSearchRequest, + coverage: "complete" | "partial" = "complete", +) { + return { + containers: [{ + canonicalPath: [ + { quoted: false, role: "catalog", value: "memory" }, + { quoted: false, role: "schema", value: "main" }, + ], + containerEntityId: `schema:${request.prefix.value}`, + detail: "Schema", + insertText: "main", + matchQuality: "exact", + }], + coverage, + epoch: request.expectedEpoch ?? epoch, + status: "ready", + }; +} + +function setup( + search: ( + request: SqlNamespaceCatalogSearchRequest, + signal: AbortSignal, + ) => unknown, + maxCacheEntries = 16, +) { + const created = createSqlNamespaceCatalogCoordinator({ + maxCacheEntries, + provider: { id: "namespaces", search }, + }); + if (created.status !== "created") throw new Error("coordinator"); + const prepared = created.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "connection", + }); + if (prepared.status !== "prepared") throw new Error("owner"); + return { + coordinator: created.coordinator, + owner: prepared.owner, + }; +} + +function deferred() { + let resolve: (value: Value) => void = (): void => {}; + let reject: (reason?: unknown) => void = (): void => {}; + const promise = new Promise((resolve_, reject_) => { + resolve = resolve_; + reject = reject_; + }); + return { promise, reject, resolve }; +} + +describe("namespace catalog coordinator", () => { + it("performs exactly one bounded search for one query site", async () => { + const searches: SqlNamespaceCatalogSearchRequest[] = []; + const { coordinator, owner } = setup((request) => { + searches.push(request); + return response(request); + }); + const outcome = await owner.request({ + ...input(), + prefix: { quoted: true, value: "ma" }, + }).result; + expect(searches).toHaveLength(1); + expect(searches[0]).toMatchObject({ + dialectId: "duckdb", + limit: 10, + prefix: { quoted: true, value: "ma" }, + qualifier: [{ value: "memory" }], + scope: "connection", + }); + expect(outcome).toMatchObject({ + providerId: "namespaces", + response: { status: "ready" }, + scope: "connection", + status: "usable", + }); + expect(Object.isFrozen(outcome)).toBe(true); + coordinator.dispose(); + }); + + it("observes cold epochs and caches complete searches", async () => { + const expected: Array = []; + const { owner } = setup((request) => { + expected.push(request.expectedEpoch); + return { ...response(request), epoch }; + }); + await owner.request(input("ma", null)).result; + const cached = await owner.request(input("ma", null)).result; + await owner.request(input("ma", nextEpoch)).result; + expect(expected).toEqual([null, nextEpoch]); + expect(cached.status).toBe("usable"); + }); + + it("does not cache partial, loading, or failed results", async () => { + let calls = 0; + const { owner } = setup((request) => { + calls += 1; + if (calls <= 2) return response(request, "partial"); + if (calls <= 4) { + return { epoch, status: "loading" }; + } + return { + code: "unavailable", + epoch, + retry: "next-request", + status: "failed", + }; + }); + await owner.request(input("partial")).result; + await owner.request(input("partial")).result; + await owner.request(input("loading")).result; + await owner.request(input("loading")).result; + await owner.request(input("failed")).result; + await owner.request(input("failed")).result; + expect(calls).toBe(6); + }); + + it("cancels and supersedes pending owner searches", async () => { + const pending: Array>> = []; + const signals: AbortSignal[] = []; + const { owner } = setup((_request, signal) => { + const work = deferred(); + pending.push(work); + signals.push(signal); + return work.promise; + }); + const first = owner.request(input("first")); + const second = owner.request(input("second")); + await expect(first.result).resolves.toEqual({ + status: "superseded", + }); + expect(signals[0]?.aborted).toBe(true); + second.cancel(); + second.cancel(); + await expect(second.result).resolves.toEqual({ + status: "cancelled", + }); + expect(signals[1]?.aborted).toBe(true); + pending[0]?.resolve({}); + pending[1]?.reject(new Error("late")); + await Promise.resolve(); + }); + + it("preserves the newest request across reentrant abort handlers", async () => { + const signals: AbortSignal[] = []; + let owner: ReturnType["owner"] | null = null; + const reentrant: { + ticket: + | ReturnType["owner"]["request"]> + | null; + } = { ticket: null }; + const configured = setup((_request, signal) => { + signals.push(signal); + if (signals.length === 1) { + signal.addEventListener("abort", () => { + reentrant.ticket = owner?.request(input("reentrant")) ?? null; + }, { once: true }); + } + return new Promise(() => undefined); + }); + owner = configured.owner; + const first = owner.request(input("first")); + const interrupted = owner.request(input("interrupted")); + + await expect(first.result).resolves.toEqual({ + status: "superseded", + }); + await expect(interrupted.result).resolves.toEqual({ + status: "superseded", + }); + expect(signals.map((signal) => signal.aborted)).toEqual([ + true, + false, + ]); + + const newest = owner.request(input("newest")); + expect(signals.map((signal) => signal.aborted)).toEqual([ + true, + true, + false, + ]); + await expect(reentrant.ticket?.result).resolves.toEqual({ + status: "superseded", + }); + newest.cancel(); + }); + + it.each(["owner", "coordinator"] as const)( + "does not resurrect work after reentrant %s disposal", + async (target) => { + const signals: AbortSignal[] = []; + let dispose = (): void => {}; + const configured = setup((_request, signal) => { + signals.push(signal); + if (signals.length === 1) { + signal.addEventListener("abort", () => dispose(), { + once: true, + }); + } + return new Promise(() => undefined); + }); + dispose = target === "owner" + ? configured.owner.dispose + : configured.coordinator.dispose; + const first = configured.owner.request(input("first")); + const interrupted = configured.owner.request(input("interrupted")); + + await expect(first.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + await expect(interrupted.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + expect(signals).toHaveLength(1); + expect(signals[0]?.aborted).toBe(true); + }, + ); + + it("isolates owners and settles disposal", async () => { + const pending: Array>> = []; + const { coordinator, owner } = setup(() => { + const work = deferred(); + pending.push(work); + return work.promise; + }); + const other = coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "connection", + }); + if (other.status !== "prepared") throw new Error("second owner"); + const first = owner.request(input("first")); + const second = other.owner.request(input("second")); + owner.dispose(); + owner.dispose(); + await expect(first.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + let secondSettled = false; + void second.result.then(() => { + secondSettled = true; + }); + await Promise.resolve(); + expect(secondSettled).toBe(false); + coordinator.dispose(); + coordinator.dispose(); + await expect(second.result).resolves.toEqual({ + reason: "disposed", + status: "unavailable", + }); + expect(coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "connection", + })).toMatchObject({ + reason: "disposed", + status: "unavailable", + }); + expect(owner.request(input())).toBeDefined(); + }); + + it("contains provider failures and malformed settlements", async () => { + const thrown = setup(() => { + throw new Error("sync"); + }); + await expect(thrown.owner.request(input()).result).resolves + .toMatchObject({ + reason: "provider-failed", + status: "unavailable", + }); + const rejected = setup(() => Promise.reject(new Error("async"))); + await expect(rejected.owner.request(input()).result).resolves + .toMatchObject({ + reason: "provider-failed", + status: "unavailable", + }); + const malformed = setup(() => ({})); + await expect(malformed.owner.request(input()).result).resolves + .toMatchObject({ + reason: "malformed-response", + status: "unavailable", + }); + }); + + it("bounds the LRU and re-fetches evicted searches", async () => { + let calls = 0; + const { owner } = setup((request) => { + calls += 1; + return response(request); + }, 1); + await owner.request(input("one")).result; + await owner.request(input("two")).result; + await owner.request(input("one")).result; + expect(calls).toBe(3); + }); + + it("fails closed for hostile or invalid configuration and inputs", async () => { + expect(createSqlNamespaceCatalogCoordinator(null)).toMatchObject({ + reason: "invalid-provider", + status: "unavailable", + }); + expect(createSqlNamespaceCatalogCoordinator({ + maxCacheEntries: 0, + provider: { id: "x", search() {} }, + })).toMatchObject({ + reason: "invalid-options", + status: "unavailable", + }); + const created = createSqlNamespaceCatalogCoordinator({ + provider: { id: "x", search() {} }, + }); + if (created.status !== "created") throw new Error("created"); + expect(created.coordinator.prepareOwner({ + dialectId: "", + scope: "", + })).toMatchObject({ + reason: "invalid-options", + status: "unavailable", + }); + const prepared = created.coordinator.prepareOwner({ + dialectId: "duckdb", + scope: "scope", + }); + if (prepared.status !== "prepared") throw new Error("prepared"); + const invalidTicket = prepared.owner.request({ + ...input(), + limit: 0, + }); + invalidTicket.cancel(); + expect(await invalidTicket.result).toMatchObject({ + reason: "invalid-request", + status: "unavailable", + }); + const hostile = {}; + Object.defineProperty(hostile, "expectedEpoch", { get: vi.fn() }); + const hostileTicket = Reflect.apply( + prepared.owner.request, + undefined, + [hostile], + ); + expect(await hostileTicket.result).toMatchObject({ + reason: "invalid-request", + status: "unavailable", + }); + }); +}); + +describe("namespace completion composer", () => { + const readyOutcome: Extract< + SqlNamespaceCatalogSearchOutcome, + { readonly status: "usable" } + > = { + providerId: "namespaces", + response: { + containers: [ + { + canonicalPath: [ + { quoted: false, role: "catalog", value: "memory" }, + { quoted: false, role: "schema", value: "main" }, + ], + containerEntityId: "main", + detail: "Schema", + insertText: "main", + matchQuality: "exact", + provenance: { + containerEntityId: "main", + epoch, + providerId: "namespaces", + scope: "connection", + }, + }, + { + canonicalPath: [ + { quoted: false, role: "project", value: "alpha" }, + { quoted: false, role: "dataset", value: "metrics" }, + ], + containerEntityId: "metrics", + insertText: "metrics", + matchQuality: "equivalent", + provenance: { + containerEntityId: "metrics", + epoch, + providerId: "namespaces", + scope: "connection", + }, + }, + ], + coverage: "complete", + epoch, + status: "ready", + }, + scope: "connection", + status: "usable", + }; + + function catalogResponse() { + if (readyOutcome.response.status !== "ready") { + throw new Error("ready response"); + } + return readyOutcome.response; + } + + it("prepares one immutable search from a query site", () => { + const prepared = prepareSqlNamespaceCatalogSearch({ + prefix: { quoted: false, value: "ma" }, + qualifier: [{ quoted: false, value: "memory" }], + replacementRange: { from: 7, to: 9 }, + }, null, [], 20); + expect(prepared).toEqual({ + expectedEpoch: null, + limit: 20, + prefix: { quoted: false, value: "ma" }, + qualifier: [{ quoted: false, value: "memory" }], + searchPaths: [], + }); + expect(Object.isFrozen(prepared)).toBe(true); + }); + + it("filters prefixes, deduplicates, orders, and uses provider edits", () => { + const catalog = catalogResponse(); + const main = catalog.containers[0]; + if (!main) throw new Error("main"); + const parent = main.canonicalPath[0]; + const child = main.canonicalPath[1]; + if (!child) throw new Error("child"); + const copied: SqlNamespaceCatalogResolvedContainer = { + ...main, + canonicalPath: [ + parent, + { ...child, quoted: true }, + ], + containerEntityId: "main-copy", + provenance: { + ...main.provenance, + containerEntityId: "main-copy", + }, + }; + const tied: SqlNamespaceCatalogResolvedContainer = { + ...main, + containerEntityId: "aaa", + provenance: { + ...main.provenance, + containerEntityId: "aaa", + }, + }; + const duplicated = { + ...readyOutcome, + response: { + ...catalog, + containers: [ + ...catalog.containers, + main, + copied, + tied, + ], + }, + }; + const all = composeSqlNamespaceCompletion({ + matchPrefix: (): "match" => "match", + outcome: duplicated, + prefix: { quoted: false, value: "" }, + providerId: "namespaces", + replacementRange: { from: 10, to: 12 }, + }); + expect(all?.value.items.map((value) => value.label)) + .toEqual(["main", "main", "main", "metrics"]); + const composition = composeSqlNamespaceCompletion({ + matchPrefix: (candidate, prefix) => + candidate.value.startsWith(prefix.value) + ? "match" + : "no-match", + outcome: duplicated, + prefix: { quoted: false, value: "ma" }, + providerId: "namespaces", + replacementRange: { from: 10, to: 12 }, + }); + expect(composition).toMatchObject({ + source: { coverage: "complete", outcome: "ready" }, + value: { + isIncomplete: false, + items: [{ + edit: { from: 10, insert: "main", to: 12 }, + label: "main", + role: "schema", + }, { + edit: { from: 10, insert: "main", to: 12 }, + label: "main", + role: "schema", + }, { + edit: { from: 10, insert: "main", to: 12 }, + label: "main", + role: "schema", + }], + }, + }); + expect(Object.isFrozen(composition?.value.items)).toBe(true); + }); + + it("surfaces partial and uncertain prefix coverage", () => { + const catalog = catalogResponse(); + const composition = composeSqlNamespaceCompletion({ + matchPrefix: () => { + throw new Error("dialect"); + }, + outcome: { + ...readyOutcome, + response: { ...catalog, coverage: "partial" }, + }, + prefix: { quoted: false, value: "m" }, + providerId: "namespaces", + replacementRange: { from: 0, to: 1 }, + }); + expect(composition?.value).toMatchObject({ + isIncomplete: true, + issues: [ + "namespace-catalog-partial", + "namespace-prefix-uncertain", + ], + items: [], + }); + }); + + it("maps loading, failed, unavailable, and cancellation", () => { + const base = { + matchPrefix: (): "match" => "match", + prefix: { quoted: false, value: "" }, + providerId: "namespaces", + replacementRange: { from: 0, to: 0 }, + }; + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { + providerId: "namespaces", + response: { epoch, status: "loading" }, + scope: "connection", + status: "usable", + }, + })).toMatchObject({ + source: { outcome: "loading" }, + value: { issues: ["namespace-catalog-loading"] }, + }); + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { + providerId: "namespaces", + response: { + code: "unknown", + epoch, + retry: "never", + status: "failed", + }, + scope: "connection", + status: "usable", + }, + })).toMatchObject({ + source: { + code: "unknown", + outcome: "failed", + retry: "never", + }, + value: { issues: ["namespace-catalog-failed"] }, + }); + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { + reason: "malformed-response", + status: "unavailable", + }, + })).toMatchObject({ + source: { + outcome: "unavailable", + reason: "malformed-response", + }, + value: { issues: ["namespace-catalog-malformed"] }, + }); + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { + reason: "provider-failed", + status: "unavailable", + }, + })).toMatchObject({ + value: { issues: ["namespace-catalog-failed"] }, + }); + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { status: "cancelled" }, + })).toBeNull(); + expect(composeSqlNamespaceCompletion({ + ...base, + outcome: { status: "superseded" }, + })).toBeNull(); + }); +}); diff --git a/src/vnext/__tests__/node-sql-parser-adapter.test.ts b/src/vnext/__tests__/node-sql-parser-adapter.test.ts index b9d61fe..e951f1f 100644 --- a/src/vnext/__tests__/node-sql-parser-adapter.test.ts +++ b/src/vnext/__tests__/node-sql-parser-adapter.test.ts @@ -5,6 +5,7 @@ import { exportedForTesting, getBigQueryNodeSqlStatementParser, getDuckDbCompatibilityNodeSqlStatementParser, + getNodeSqlParserQueryBindingModel, getPostgresqlNodeSqlStatementParser, MAX_NODE_SQL_PARSER_STATEMENT_LENGTH, } from "../node-sql-parser-adapter.js"; @@ -885,6 +886,20 @@ describe("real node-sql-parser builds", () => { mode: "compatibility", status: "parsed", }); + if ( + expectedKind === "query" && + state.analysis.status === "parsed" + ) { + expect( + getNodeSqlParserQueryBindingModel(state.analysis.artifact), + ).toMatchObject({ + coverage: { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }, + }); + } }, ); @@ -935,6 +950,18 @@ describe("real node-sql-parser builds", () => { mode: "compatibility", status: "parsed", }); + if (state.analysis.status === "parsed") { + expect( + getNodeSqlParserQueryBindingModel(state.analysis.artifact), + ).toMatchObject({ + coverage: { + queryBlocks: "partial", + relationBindings: "partial", + visibility: "partial", + }, + issues: [{ code: "parser-compatibility" }], + }); + } }); it("never turns a DuckDB compatibility rejection into invalid SQL", async () => { @@ -953,7 +980,7 @@ describe("real node-sql-parser builds", () => { }); describe("backend artifact privacy", () => { - it("retains backend data only behind the authentic artifact", async () => { + it("retains only normalized bindings behind the authentic artifact", async () => { const backendRoot = { privateBackendField: "must-not-leak", type: "select", @@ -966,7 +993,15 @@ describe("backend artifact privacy", () => { } const { artifact } = state.analysis; - expect(exportedForTesting.hasBackendPayload(artifact)).toBe(true); + expect(exportedForTesting.hasBackendPayload(artifact)).toBe(false); + expect(getNodeSqlParserQueryBindingModel(artifact)).toMatchObject({ + coverage: { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }, + statementRange: { from: 0, to: 8 }, + }); expect(Object.keys(artifact)).toEqual(["kind", "range"]); expect(JSON.stringify(state)).not.toContain("must-not-leak"); expect(Reflect.ownKeys(artifact)).not.toContain("privateBackendField"); @@ -981,6 +1016,12 @@ describe("backend artifact privacy", () => { structuralCopy, ), ).toBe(false); + expect( + invokeWithUnknown( + getNodeSqlParserQueryBindingModel, + structuralCopy, + ), + ).toBeNull(); }); it("does not retain malformed or unsupported backend values", async () => { diff --git a/src/vnext/__tests__/node-sql-parser-browser-executor.test.ts b/src/vnext/__tests__/node-sql-parser-browser-executor.test.ts index 9e86078..20fc812 100644 --- a/src/vnext/__tests__/node-sql-parser-browser-executor.test.ts +++ b/src/vnext/__tests__/node-sql-parser-browser-executor.test.ts @@ -294,7 +294,11 @@ function ready(worker: FakeWorker): void { function respond( worker: FakeWorker, outcome: - | { readonly kind: "parsed"; readonly statementKind: "query" } + | { + readonly kind: "parsed"; + readonly queryBindings: null; + readonly statementKind: "query"; + } | { readonly kind: "syntax-rejected" } | { readonly kind: "unsupported"; @@ -309,8 +313,9 @@ function respond( const backendOutcome = outcome.kind === "parsed" ? { - ...outcome, + kind: outcome.kind, root: Object.freeze({}), + statementKind: outcome.statementKind, } : outcome.kind === "failed" ? { @@ -322,6 +327,7 @@ function respond( encodeNodeSqlParserWireBackendOutcome( postedRequest(worker, index).requestId, backendOutcome, + outcome.kind === "parsed" ? outcome.queryBindings : null, ), ); } @@ -366,7 +372,7 @@ describe("node-sql-parser browser executor admission", () => { expect(request).toStrictEqual({ grammar: "postgresql", kind: "parse", - protocolVersion: 1, + protocolVersion: 2, requestId: 1, text: "SELECT 1", }); @@ -374,11 +380,13 @@ describe("node-sql-parser browser executor admission", () => { respond(worker, { kind: "parsed", + queryBindings: null, statementKind: "query", }); const result = await outcome(submission); expect(result).toStrictEqual({ kind: "parsed", + queryBindings: null, statementKind: "query", }); expect(Object.isFrozen(result)).toBe(true); @@ -429,10 +437,12 @@ describe("node-sql-parser browser executor admission", () => { respond(worker, { kind: "parsed", + queryBindings: null, statementKind: "query", }, 2); expect(await outcome(third)).toStrictEqual({ kind: "parsed", + queryBindings: null, statementKind: "query", }); }); @@ -1210,12 +1220,12 @@ describe("node-sql-parser browser executor hostile worker handling", () => { null, {}, { data: null }, - { data: { kind: "ready", protocolVersion: 2 } }, + { data: { kind: "ready", protocolVersion: 1 } }, { data: { extra: true, kind: "ready", - protocolVersion: 1, + protocolVersion: 2, }, }, ])("fails closed for malformed message event %#", async (event) => { @@ -1285,7 +1295,7 @@ describe("node-sql-parser browser executor hostile worker handling", () => { worker.emit({ code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, } satisfies NodeSqlParserWireMessage); expect(await outcome(submission)).toStrictEqual({ code: "protocol-error", @@ -1311,7 +1321,7 @@ describe("node-sql-parser browser executor hostile worker handling", () => { firstWorker.emit({ kind: "syntax-rejected", - protocolVersion: 1, + protocolVersion: 2, requestId: activeId + 1, } satisfies NodeSqlParserWireMessage); expect(await outcome(active)).toStrictEqual({ diff --git a/src/vnext/__tests__/node-sql-parser-browser-worker-endpoint.test.ts b/src/vnext/__tests__/node-sql-parser-browser-worker-endpoint.test.ts index 749c469..f4a516e 100644 --- a/src/vnext/__tests__/node-sql-parser-browser-worker-endpoint.test.ts +++ b/src/vnext/__tests__/node-sql-parser-browser-worker-endpoint.test.ts @@ -161,7 +161,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.listenerCount()).toBe(1); expect(scope.messages).toStrictEqual([ - { kind: "ready", protocolVersion: 1 }, + { kind: "ready", protocolVersion: 2 }, ]); expect(evaluations).toStrictEqual({ bigquery: 0, @@ -172,7 +172,8 @@ describe("node-sql-parser browser worker endpoint", () => { await waitForMessageCount(scope, 2); expect(scope.messages[1]).toStrictEqual({ kind: "parsed", - protocolVersion: 1, + queryBindings: expect.any(Object), + protocolVersion: 2, requestId: 1, statementKind: "query", }); @@ -185,7 +186,8 @@ describe("node-sql-parser browser worker endpoint", () => { await waitForMessageCount(scope, 3); expect(scope.messages[2]).toStrictEqual({ kind: "parsed", - protocolVersion: 1, + queryBindings: expect.any(Object), + protocolVersion: 2, requestId: 2, statementKind: "insert", }); @@ -330,7 +332,8 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ kind: "parsed", - protocolVersion: 1, + queryBindings: expect.any(Object), + protocolVersion: 2, requestId: 19, statementKind: "query", }); @@ -374,7 +377,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "module-load", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 17, }); expect(JSON.stringify(scope.messages[1])).not.toContain( @@ -420,7 +423,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 1, }); expect(scope.closeCalls()).toBe(1); @@ -457,7 +460,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 8, }); expect(scope.closeCalls()).toBe(1); @@ -508,7 +511,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 10, }); expect(scope.closeCalls()).toBe(1); @@ -547,7 +550,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 11, }); expect(JSON.stringify(scope.messages[1])).not.toContain( @@ -589,7 +592,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 9, }); expect(evaluations).toBe(0); @@ -623,7 +626,7 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, }); expect(scope.closeCalls()).toBe(1); expect(scope.listenerCount()).toBe(0); @@ -645,11 +648,11 @@ describe("node-sql-parser browser worker endpoint", () => { scope.dispatch({ ...valid, extra: true }); expect(scope.messages).toStrictEqual([ - { kind: "ready", protocolVersion: 1 }, + { kind: "ready", protocolVersion: 2 }, { code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, }, ]); expect(scope.closeCalls()).toBe(1); @@ -678,7 +681,8 @@ describe("node-sql-parser browser worker endpoint", () => { expect(scope.messages[1]).toStrictEqual({ kind: "parsed", - protocolVersion: 1, + queryBindings: expect.any(Object), + protocolVersion: 2, requestId: 23, statementKind: "query", }); @@ -694,7 +698,7 @@ describe("node-sql-parser browser worker endpoint", () => { code: undefined, expected: { kind: "syntax-rejected", - protocolVersion: 1, + protocolVersion: 2, requestId: 4, }, moduleValue: parserModule(() => { @@ -712,7 +716,7 @@ describe("node-sql-parser browser worker endpoint", () => { code: undefined, expected: { kind: "unsupported", - protocolVersion: 1, + protocolVersion: 2, reason: "multiple-statements", requestId: 4, }, @@ -726,7 +730,7 @@ describe("node-sql-parser browser worker endpoint", () => { expected: { code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 4, }, moduleValue: parserModule(() => { @@ -738,7 +742,7 @@ describe("node-sql-parser browser worker endpoint", () => { expected: { code: "malformed-output", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 4, }, moduleValue: {}, @@ -863,11 +867,11 @@ describe("node-sql-parser browser worker endpoint", () => { scope.dispatch(request(1)); expect(scope.messages).toStrictEqual([ - { kind: "ready", protocolVersion: 1 }, + { kind: "ready", protocolVersion: 2 }, { code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, }, ]); expect(scope.closeCalls()).toBe(1); diff --git a/src/vnext/__tests__/node-sql-parser-query-bindings.bench.ts b/src/vnext/__tests__/node-sql-parser-query-bindings.bench.ts new file mode 100644 index 0000000..ca3c991 --- /dev/null +++ b/src/vnext/__tests__/node-sql-parser-query-bindings.bench.ts @@ -0,0 +1,31 @@ +import { bench, describe } from "vitest"; +import { + normalizeNodeSqlParserQueryBindings, +} from "../node-sql-parser-query-bindings.js"; + +const RELATION_COUNT = 1_000; +const relations = Array.from( + { length: RELATION_COUNT }, + (_, index) => `t${index} a${index}`, +); +const text = `SELECT a999.id FROM ${relations.join(", ")}`; +const root = { + from: Array.from({ length: RELATION_COUNT }, (_, index) => ({ + as: `a${index}`, + db: null, + table: `t${index}`, + })), + type: "select", +}; +const authority = {}; + +describe("node-sql-parser query binding normalization", () => { + bench("normalizes 1,000 relations", () => { + normalizeNodeSqlParserQueryBindings( + root, + text, + authority, + { compatibility: false, grammar: "postgresql" }, + ); + }); +}); diff --git a/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts b/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts new file mode 100644 index 0000000..24b2c7f --- /dev/null +++ b/src/vnext/__tests__/node-sql-parser-query-bindings.test.ts @@ -0,0 +1,1291 @@ +// @vitest-environment node + +import { describe, expect, it } from "vitest"; +import { + normalizeNodeSqlParserQueryBindings, + type NodeSqlParserQueryBindingOptions, +} from "../node-sql-parser-query-bindings.js"; +import { + resolveSqlRelationQualifier, + visibleSqlRelationBindingsAt, +} from "../query-binding-model.js"; +import { + decodeNodeSqlParserWireMessage, + encodeNodeSqlParserWireBackendOutcome, +} from "../node-sql-parser-wire.js"; +import type { SqlIdentifierComponent } from "../types.js"; + +const PG: NodeSqlParserQueryBindingOptions = { + compatibility: false, + grammar: "postgresql", +}; +const BQ: NodeSqlParserQueryBindingOptions = { + compatibility: false, + grammar: "bigquery", +}; + +function normalize( + root: unknown, + text: string, + options: NodeSqlParserQueryBindingOptions = PG, +) { + return normalizeNodeSqlParserQueryBindings(root, text, {}, options); +} + +function relation(table: string, alias: string | null = null) { + return { as: alias, db: null, table }; +} + +function equals( + left: SqlIdentifierComponent, + right: SqlIdentifierComponent, +): boolean { + return left.quoted === right.quoted && + (left.quoted + ? left.value === right.value + : left.value.toLowerCase() === right.value.toLowerCase()); +} + +describe("node-sql-parser query binding normalization", () => { + it("normalizes named relations, aliases, scopes, and clause visibility", () => { + const text = + "SELECT u.id FROM users AS u JOIN teams t ON u.team_id=t.id " + + "WHERE u.active GROUP BY u.id HAVING count(*)>1 " + + "ORDER BY u.id LIMIT 5"; + const result = normalize( + { + from: [ + relation("users", "u"), + { ...relation("teams", "t"), join: "JOIN", on: {} }, + ], + type: "select", + where: {}, + }, + text, + ); + + expect(result.status).toBe("ready"); + if (result.status !== "ready") { + return; + } + expect(result.model.coverage).toEqual({ + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }); + expect(result.model.bindings).toMatchObject([ + { + alias: { explicit: true, name: { value: "u" } }, + source: { kind: "named", path: [{ value: "users" }] }, + }, + { + alias: { explicit: false, name: { value: "t" } }, + source: { kind: "named", path: [{ value: "teams" }] }, + }, + ]); + expect(result.model.regions.map((region) => region.kind)).toEqual( + expect.arrayContaining([ + "select-list", + "from-source", + "join-condition", + "where", + "group-by", + "having", + "order-by", + "limit", + ]), + ); + const selected = visibleSqlRelationBindingsAt( + result.model, + text.indexOf("u.id"), + ); + expect(selected.status === "ready" && selected.bindings).toHaveLength(2); + expect( + resolveSqlRelationQualifier( + result.model, + text.indexOf("u.id"), + { quoted: false, value: "U" }, + equals, + ).status, + ).toBe("resolved"); + }); + + it("normalizes derived query blocks without leaking sibling scope", () => { + const text = + "SELECT d.id FROM (SELECT id FROM teams) AS d " + + "JOIN users u ON d.id=u.id"; + const derived = { + from: [relation("teams")], + type: "select", + }; + const result = normalize( + { + from: [ + { + as: "d", + expr: { ast: derived, parentheses: true }, + }, + { ...relation("users", "u"), join: "JOIN", on: {} }, + ], + type: "select", + }, + text, + ); + + expect(result.status).toBe("ready"); + if (result.status !== "ready") { + return; + } + expect(result.model.blocks).toHaveLength(2); + expect(result.model.blocks[1]?.parentBlock).toBe(0); + expect(result.model.bindings).toMatchObject([ + { alias: { name: { value: "d" } }, source: { block: 1, kind: "derived" } }, + { alias: { name: { value: "u" } }, source: { kind: "named" } }, + { owner: 1, source: { kind: "named", path: [{ value: "teams" }] } }, + ]); + expect(result.model.blocks[1]?.baseScope).toBe(0); + }); + + it("resolves CTE references to declaration evidence", () => { + const text = + "WITH recent AS (SELECT * FROM events) " + + "SELECT * FROM recent r"; + const cteStatement = { + from: [relation("events")], + type: "select", + }; + const result = normalize( + { + from: [relation("recent", "r")], + type: "select", + with: [ + { + name: { type: "default", value: "recent" }, + stmt: cteStatement, + }, + ], + }, + text, + ); + + expect(result.status).toBe("ready"); + if (result.status === "ready") { + expect(result.model.blocks).toHaveLength(2); + expect(result.model.bindings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + source: expect.objectContaining({ + kind: "cte", + name: { quoted: false, value: "recent" }, + }), + }), + ]), + ); + } + }); + + it("preserves quoted CTE identity and case semantics", () => { + const cteStatement = { + from: [relation("events")], + type: "select", + }; + const quoted = normalize( + { + from: [relation("Recent")], + type: "select", + with: [ + { + name: { type: "default", value: "Recent" }, + stmt: cteStatement, + }, + ], + }, + 'WITH "Recent" AS (SELECT * FROM events) SELECT * FROM "Recent"', + ); + expect(quoted).toMatchObject({ + model: { + bindings: [ + { source: { kind: "cte", name: { quoted: true, value: "Recent" } } }, + { source: { kind: "named" } }, + ], + }, + status: "ready", + }); + + const differentlyQuoted = normalize( + { + from: [relation("recent")], + type: "select", + with: [ + { + name: { type: "default", value: "Recent" }, + stmt: cteStatement, + }, + ], + }, + 'WITH "Recent" AS (SELECT * FROM events) SELECT * FROM recent', + ); + expect(differentlyQuoted).toMatchObject({ + model: { + bindings: [ + { source: { kind: "named", path: [{ value: "recent" }] } }, + { source: { kind: "named" } }, + ], + }, + status: "ready", + }); + }); + + it("applies parent and preceding-sibling CTE visibility per block", () => { + const first = { + from: [relation("later")], + type: "select", + }; + const later = { + from: [relation("events")], + type: "select", + }; + const result = normalize( + { + from: [relation("first")], + type: "select", + with: [ + { name: { value: "first" }, stmt: first }, + { name: { value: "later" }, stmt: later }, + ], + }, + "WITH first AS (SELECT * FROM later), " + + "later AS (SELECT * FROM events) SELECT * FROM first", + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte", name: { value: "first" } } }, + { + owner: 1, + source: { kind: "named", path: [{ value: "later" }] }, + }, + { + owner: 2, + source: { kind: "named", path: [{ value: "events" }] }, + }, + ], + }, + status: "ready", + }); + }); + + it("authenticates recursive CTE self-visibility", () => { + const body = { + from: [relation("self")], + type: "select", + }; + const result = normalize( + { + from: [relation("self")], + type: "select", + with: [{ + name: { value: "self" }, + recursive: true, + stmt: body, + }], + }, + "WITH RECURSIVE self AS (SELECT * FROM self) SELECT * FROM self", + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte", name: { value: "self" } } }, + { owner: 1, source: { kind: "cte", name: { value: "self" } } }, + ], + coverage: { relationBindings: "complete" }, + }, + status: "ready", + }); + if (result.status === "ready") { + expect(result.model.issues).not.toContainEqual( + expect.objectContaining({ code: "recursive-cte-uncertainty" }), + ); + } + }); + + it("fails closed when recursive lexical and AST evidence disagree", () => { + const body = { + from: [relation("self")], + type: "select", + }; + const lexicalOnly = normalize( + { + from: [relation("self")], + type: "select", + with: [{ name: { value: "self" }, stmt: body }], + }, + "WITH RECURSIVE self AS (SELECT * FROM self) SELECT * FROM self", + ); + const astOnly = normalize( + { + from: [relation("self")], + type: "select", + with: [{ + name: { value: "self" }, + recursive: true, + stmt: body, + }], + }, + "WITH self AS (SELECT * FROM self) SELECT * FROM self", + ); + for (const result of [lexicalOnly, astOnly]) { + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte" } }, + { + owner: 1, + source: { kind: "unknown", reason: "unknown-correlation" }, + }, + ], + coverage: { relationBindings: "partial" }, + issues: [{ code: "recursive-cte-uncertainty" }], + }, + status: "ready", + }); + } + }); + + it("does not resolve an uncertain local CTE through an outer namesake", () => { + const nested = { + from: [relation("shadow")], + type: "select", + with: [{ + name: { value: "shadow" }, + stmt: { from: [relation("shadow")], type: "select" }, + }], + }; + const result = normalize( + { + from: [{ as: "d", expr: { ast: nested, parentheses: true } }], + type: "select", + with: [{ + name: { value: "shadow" }, + stmt: { from: [relation("events")], type: "select" }, + }], + }, + "WITH shadow AS (SELECT * FROM events) SELECT * FROM (" + + "WITH RECURSIVE shadow AS (SELECT * FROM shadow) " + + "SELECT * FROM shadow) d", + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { block: 2, kind: "derived" } }, + { + owner: 1, + source: { kind: "named", path: [{ value: "events" }] }, + }, + { owner: 2, source: { kind: "cte", name: { value: "shadow" } } }, + { + owner: 3, + source: { kind: "unknown", reason: "unknown-correlation" }, + }, + ], + coverage: { relationBindings: "partial" }, + issues: [{ code: "recursive-cte-uncertainty" }], + }, + status: "ready", + }); + }); + + it("keeps non-recursive self and forward names outside CTE visibility", () => { + const self = normalize( + { + from: [relation("self")], + type: "select", + with: [{ + name: { value: "self" }, + stmt: { from: [relation("self")], type: "select" }, + }], + }, + "WITH self AS (SELECT * FROM self) SELECT * FROM self", + ); + expect(self).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte" } }, + { owner: 1, source: { kind: "named", path: [{ value: "self" }] } }, + ], + coverage: { relationBindings: "complete" }, + }, + status: "ready", + }); + }); + + it("marks recursive forward and mutual references uncertain", () => { + const first = { + from: [relation("later")], + type: "select", + }; + const forward = normalize( + { + from: [relation("first")], + type: "select", + with: [ + { + name: { value: "first" }, + recursive: true, + stmt: first, + }, + { + name: { value: "later" }, + stmt: { from: [relation("events")], type: "select" }, + }, + ], + }, + "WITH RECURSIVE first AS (SELECT * FROM later), " + + "later AS (SELECT * FROM events) SELECT * FROM first", + ); + expect(forward).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte", name: { value: "first" } } }, + { + owner: 1, + source: { kind: "unknown", reason: "unknown-correlation" }, + }, + { + owner: 2, + source: { kind: "named", path: [{ value: "events" }] }, + }, + ], + coverage: { relationBindings: "partial" }, + issues: [{ code: "recursive-cte-uncertainty" }], + }, + status: "ready", + }); + + const later = { + from: [relation("first")], + type: "select", + }; + const result = normalize( + { + from: [relation("first")], + type: "select", + with: [ + { + name: { value: "first" }, + recursive: true, + stmt: first, + }, + { name: { value: "later" }, stmt: later }, + ], + }, + "WITH RECURSIVE first AS (SELECT * FROM later), " + + "later AS (SELECT * FROM first) SELECT * FROM first", + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { kind: "cte", name: { value: "first" } } }, + { + owner: 1, + source: { kind: "unknown", reason: "unknown-correlation" }, + }, + { owner: 2, source: { kind: "cte", name: { value: "first" } } }, + ], + coverage: { relationBindings: "partial" }, + issues: [{ code: "recursive-cte-uncertainty" }], + }, + status: "ready", + }); + }); + + it("resolves a nested WITH inside its derived query block", () => { + const nestedCte = { + from: [relation("events")], + type: "select", + }; + const nested = { + from: [relation("local")], + type: "select", + with: [{ name: { value: "local" }, stmt: nestedCte }], + }; + const result = normalize( + { + from: [{ as: "d", expr: { ast: nested, parentheses: true } }], + type: "select", + }, + "SELECT * FROM (WITH local AS (SELECT * FROM events) " + + "SELECT * FROM local) d", + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { owner: 0, source: { block: 1, kind: "derived" } }, + { owner: 1, source: { kind: "cte", name: { value: "local" } } }, + { + owner: 2, + source: { kind: "named", path: [{ value: "events" }] }, + }, + ], + }, + status: "ready", + }); + }); + + it("normalizes BigQuery multipart paths and QUALIFY", () => { + const text = + "SELECT a.id FROM `project.dataset.users` a " + + "QUALIFY row_number() over()=1"; + const result = normalize( + { + from: [ + { + as: "a", + db: null, + surround: { table: "`" }, + table: "project.dataset.users", + }, + ], + qualify: {}, + type: "select", + }, + text, + BQ, + ); + + expect(result.status).toBe("ready"); + if (result.status === "ready") { + expect(result.model.bindings[0]?.source).toMatchObject({ + kind: "named", + path: [ + { value: "project" }, + { value: "dataset" }, + { value: "users" }, + ], + }); + expect( + result.model.regions.some((region) => region.kind === "qualify"), + ).toBe(true); + } + }); + + it("marks DuckDB compatibility evidence partial in every dimension", () => { + const result = normalize( + { from: [relation("events")], type: "select" }, + "SELECT * FROM events", + { compatibility: true, grammar: "postgresql" }, + ); + expect(result).toMatchObject({ + model: { + coverage: { + queryBlocks: "partial", + relationBindings: "partial", + visibility: "partial", + }, + issues: [{ code: "parser-compatibility" }], + }, + status: "ready", + }); + }); + + it("downgrades unmatched lexical or parser relations without guessing", () => { + const result = normalize( + { + from: [ + relation("users"), + { as: null, expr: { type: "function" } }, + ], + type: "select", + }, + "SELECT * FROM users", + ); + expect(result).toMatchObject({ + model: { + coverage: { + relationBindings: "partial", + visibility: "partial", + }, + issues: [{ code: "unsupported-relation-source" }], + }, + status: "ready", + }); + }); + + it("marks extra lexical query blocks and unknown relation shapes partial", () => { + const extraBlock = normalize( + { type: "select" }, + "SELECT EXISTS (SELECT 1)", + ); + expect(extraBlock).toMatchObject({ + model: { + coverage: { queryBlocks: "partial" }, + issues: [{ code: "unsupported-clause" }], + }, + status: "ready", + }); + + const unknownRelation = normalize( + { from: [{}], type: "select" }, + "SELECT * FROM users", + ); + expect(unknownRelation).toMatchObject({ + model: { + coverage: { relationBindings: "partial" }, + bindings: [ + { + source: { + kind: "unknown", + reason: "unsupported-relation-source", + }, + }, + ], + }, + status: "ready", + }); + }); + + it("downgrades alias disagreement instead of trusting either side", () => { + const lexicalOnly = normalize( + { from: [relation("users")], type: "select" }, + "SELECT * FROM users u", + ); + const astOnly = normalize( + { from: [relation("users", "u")], type: "select" }, + "SELECT * FROM users", + ); + const differentValues = normalize( + { from: [relation("users", "x")], type: "select" }, + "SELECT * FROM users u", + ); + const differentQuotes = normalize( + { from: [relation("users", "U")], type: "select" }, + 'SELECT * FROM users AS "u"', + ); + for (const result of [ + lexicalOnly, + astOnly, + differentValues, + differentQuotes, + ]) { + expect(result).toMatchObject({ + model: { + bindings: [{ alias: null }], + coverage: { relationBindings: "partial" }, + issues: [{ code: "unsupported-relation-source" }], + }, + status: "ready", + }); + } + }); + + it("never publishes AST relation names absent from the SQL text", () => { + const table = normalize( + { from: [relation("events")], type: "select" }, + "SELECT * FROM users", + ); + const schema = normalize( + { + from: [{ as: null, db: "private", table: "users" }], + type: "select", + }, + "SELECT * FROM public.users", + ); + for (const result of [table, schema]) { + expect(result).toMatchObject({ + model: { + bindings: [ + { + source: { + kind: "unknown", + reason: "unsupported-relation-source", + }, + }, + ], + coverage: { relationBindings: "partial" }, + issues: [{ code: "unsupported-relation-source" }], + }, + status: "ready", + }); + } + expect( + normalize( + { from: [relation("private.dataset.users")], type: "select" }, + "SELECT * FROM `public.dataset.users`", + BQ, + ), + ).toMatchObject({ + model: { + bindings: [{ source: { kind: "unknown" } }], + coverage: { relationBindings: "partial" }, + }, + status: "ready", + }); + }); + + it("keeps ON and USING join visibility aligned with relation sites", () => { + const onText = "SELECT a.id, b.id FROM a JOIN b ON a.id=b.id"; + const onResult = normalize( + { + from: [ + relation("a"), + { ...relation("b"), join: "JOIN", on: {} }, + ], + type: "select", + }, + onText, + ); + const usingText = "SELECT * FROM a JOIN b USING (id)"; + const usingResult = normalize( + { + from: [ + relation("a"), + { ...relation("b"), join: "JOIN" }, + ], + type: "select", + }, + usingText, + ); + for (const [result, position] of [ + [onResult, onText.indexOf("a.id", onText.indexOf("ON"))], + [usingResult, usingText.indexOf("id")], + ] as const) { + expect(result.status).toBe("ready"); + if (result.status === "ready") { + expect(result.model.coverage.visibility).toBe("complete"); + expect(visibleSqlRelationBindingsAt(result.model, position)).toMatchObject({ + bindings: [{}, {}], + region: { kind: "join-condition" }, + status: "ready", + }); + } + } + }); + + it("does not assign a later join condition to an earlier relation", () => { + const text = "SELECT * FROM a JOIN b JOIN c ON c.id=b.id"; + const result = normalize( + { + from: [ + relation("a"), + { ...relation("b"), join: "JOIN" }, + { ...relation("c"), join: "JOIN", on: {} }, + ], + type: "select", + }, + text, + ); + expect(result.status).toBe("ready"); + if (result.status === "ready") { + const regions = result.model.regions.filter((region) => + region.kind === "join-condition" + ); + expect(regions).toHaveLength(1); + expect(visibleSqlRelationBindingsAt( + result.model, + text.indexOf("c.id"), + )).toMatchObject({ + bindings: [{}, {}, {}], + status: "ready", + }); + } + }); + + it.each([ + ["WHERE true", { where: {} }], + ["GROUP BY a.id", { groupby: [{ expr: {} }] }], + ["HAVING true", { having: {} }], + ["QUALIFY true", { qualify: {} }], + ["ORDER BY a.id", { orderby: [{ expr: {} }] }], + ["LIMIT 1", { limit: {} }], + ])( + "stops a conditionless join before %s", + (suffix, clause) => { + const result = normalize( + { + ...clause, + from: [ + relation("a"), + { ...relation("b"), join: "CROSS JOIN" }, + ], + type: "select", + }, + `SELECT * FROM a CROSS JOIN b ${suffix}`, + ); + expect(result.status).toBe("ready"); + if (result.status === "ready") { + expect( + result.model.regions.some((region) => + region.kind === "join-condition" + ), + ).toBe(false); + } + }, + ); + + it.each([ + "WHERE true", + "GROUP BY a.id", + "HAVING true", + "QUALIFY true", + "ORDER BY a.id", + "LIMIT 1", + ])("ends an ON region before %s", (suffix) => { + const text = `SELECT * FROM a JOIN b ON true ${suffix}`; + const result = normalize( + { + from: [ + relation("a"), + { ...relation("b"), join: "JOIN", on: {} }, + ], + type: "select", + }, + text, + ); + expect(result.status).toBe("ready"); + if (result.status === "ready") { + const join = result.model.regions.find((region) => + region.kind === "join-condition" + ); + expect(join?.range.to).toBe( + text.indexOf(suffix.split(" ")[0] ?? suffix), + ); + } + }); + + it("emits compound query block kinds", () => { + for (const type of ["union", "intersect", "except"]) { + expect(normalize({ type }, "SELECT 1")).toMatchObject({ + model: { blocks: [{ kind: "compound" }] }, + status: "ready", + }); + } + }); + + it("preserves PostgreSQL quoted path components", () => { + const result = normalize( + { + from: [{ as: "U", db: "MySchema", table: "Users" }], + type: "select", + }, + 'SELECT * FROM "MySchema"."Users" AS "U"', + ); + expect(result).toMatchObject({ + model: { + bindings: [ + { + alias: { name: { quoted: true, value: "U" } }, + source: { + kind: "named", + path: [ + { quoted: true, value: "MySchema" }, + { quoted: true, value: "Users" }, + ], + }, + }, + ], + }, + status: "ready", + }); + }); + + it("round trips normalized data across wire v2 without raw AST or text", () => { + const text = "SELECT u.id FROM users u"; + const result = normalize( + { from: [relation("users", "u")], type: "select" }, + text, + ); + if (result.status !== "ready") { + throw new Error("Expected normalized query bindings"); + } + const root = { privateAstSecret: "never-cross-wire", type: "select" }; + const encoded = encodeNodeSqlParserWireBackendOutcome( + 1, + { kind: "parsed", root, statementKind: "query" }, + result.model, + ); + const decoded = decodeNodeSqlParserWireMessage(encoded); + + expect(decoded).not.toBeNull(); + expect(decoded?.kind).toBe("parsed"); + if (decoded?.kind === "parsed") { + expect(decoded.queryBindings).toStrictEqual(result.model); + expect(decoded.queryBindings).not.toBe(result.model); + } + expect(JSON.stringify(encoded)).not.toContain("privateAstSecret"); + expect(JSON.stringify(encoded)).not.toContain(text); + expect(Object.hasOwn(encoded, "root")).toBe(false); + }); +}); + +describe("node-sql-parser query binding hostile boundaries", () => { + it("does not invoke AST accessors", () => { + let calls = 0; + const root = {}; + Object.defineProperty(root, "type", { + get() { + calls += 1; + return "select"; + }, + }); + expect(normalize(root, "SELECT 1")).toEqual({ + reason: "not-query", + status: "unavailable", + }); + expect(calls).toBe(0); + }); + + it("rejects cycles and sparse or oversized AST arrays", () => { + const cyclic = { from: [] as unknown[], type: "select" }; + cyclic.from.push({ expr: { ast: cyclic } }); + expect(normalize(cyclic, "SELECT * FROM (SELECT 1) x")).toEqual({ + reason: "malformed-ast", + status: "unavailable", + }); + + const sparse: unknown[] = []; + sparse.length = 1; + expect(normalize({ from: sparse, type: "select" }, "SELECT 1")).toEqual({ + reason: "malformed-ast", + status: "unavailable", + }); + + const oversized = Array.from({ length: 1_025 }, () => relation("x")); + expect(normalize({ from: oversized, type: "select" }, "SELECT 1")).toEqual({ + reason: "malformed-ast", + status: "unavailable", + }); + + const revoked = Proxy.revocable([], {}); + revoked.revoke(); + expect( + normalize({ from: revoked.proxy, type: "select" }, "SELECT 1"), + ).toEqual({ + reason: "malformed-ast", + status: "unavailable", + }); + }); + + it("rejects malformed known AST properties and CTE entries", () => { + const cases: readonly { + readonly root: unknown; + readonly text: string; + }[] = [ + { root: { from: {}, type: "select" }, text: "SELECT 1" }, + { root: { from: [null], type: "select" }, text: "SELECT 1" }, + { root: { type: "select", with: {} }, text: "SELECT 1" }, + { root: { type: "select", with: [null] }, text: "SELECT 1" }, + { + root: { type: "select", with: [{}] }, + text: "WITH x AS (SELECT 1) SELECT 1", + }, + { + root: { + type: "select", + with: [{ name: {}, stmt: { type: "select" } }], + }, + text: "WITH x AS (SELECT 1) SELECT 1", + }, + { + root: { + type: "select", + with: [{ name: "missing", stmt: { type: "select" } }], + }, + text: "WITH present AS (SELECT 1) SELECT 1", + }, + ]; + for (const { root, text } of cases) { + expect(normalize(root, text).status).toBe("unavailable"); + } + }); + + it("rejects CTE structures that change across safe inspections", () => { + const child = { type: "select" }; + const item = { name: "x", stmt: child }; + function changingRoot(secondWith: unknown) { + let reads = 0; + return new Proxy( + { type: "select", with: [item] }, + { + getOwnPropertyDescriptor(target, key) { + const descriptor = Reflect.getOwnPropertyDescriptor(target, key); + if (key !== "with" || descriptor === undefined) { + return descriptor; + } + reads += 1; + return { ...descriptor, value: reads === 1 ? [item] : secondWith }; + }, + }, + ); + } + for (const root of [changingRoot({}), changingRoot([])]) { + expect( + normalize(root, "WITH x AS (SELECT 1) SELECT 1"), + ).toEqual({ reason: "malformed-ast", status: "unavailable" }); + } + + let itemReads = 0; + const changingItems = new Proxy([item], { + getOwnPropertyDescriptor(target, key) { + const descriptor = Reflect.getOwnPropertyDescriptor(target, key); + if (key !== "0" || descriptor === undefined) { + return descriptor; + } + itemReads += 1; + return { ...descriptor, value: itemReads === 1 ? item : null }; + }, + }); + expect( + normalize( + { type: "select", with: changingItems }, + "WITH x AS (SELECT 1) SELECT 1", + ), + ).toEqual({ reason: "malformed-ast", status: "unavailable" }); + + function changingChild() { + let reads = 0; + return new Proxy( + { type: "select", with: [] }, + { + getOwnPropertyDescriptor(target, key) { + const descriptor = Reflect.getOwnPropertyDescriptor(target, key); + if (key !== "with" || descriptor === undefined) { + return descriptor; + } + reads += 1; + return { ...descriptor, value: reads === 1 ? [] : {} }; + }, + }, + ); + } + expect( + normalize( + { + type: "select", + with: [{ name: "x", stmt: changingChild() }], + }, + "WITH x AS (SELECT 1) SELECT 1", + ), + ).toEqual({ reason: "malformed-ast", status: "unavailable" }); + expect( + normalize( + { + from: [{ as: "d", expr: { ast: changingChild() } }], + type: "select", + }, + "SELECT * FROM (SELECT 1) d", + ), + ).toEqual({ reason: "malformed-ast", status: "unavailable" }); + }); + + it("handles comments, incomplete relation sites, and nesting limits", () => { + expect( + normalize( + { from: [relation("users")], type: "select" }, + "SELECT /* private */ * FROM users -- tail", + ).status, + ).toBe("ready"); + expect( + normalize({ type: "select" }, "SELECT * FROM").status, + ).toBe("ready"); + expect( + normalize( + { type: "select" }, + `${"(".repeat(257)}SELECT 1${")".repeat(257)}`, + ), + ).toEqual({ reason: "resource-limit", status: "unavailable" }); + }); + + it("normalizes descriptor traps and invalid normalization authority", () => { + const hostile = new Proxy( + { type: "select" }, + { + getOwnPropertyDescriptor() { + throw new Error("private trap"); + }, + }, + ); + expect(normalize(hostile, "SELECT 1")).toEqual({ + reason: "not-query", + status: "unavailable", + }); + expect( + normalizeNodeSqlParserQueryBindings( + { type: "select" }, + "SELECT 1", + null, + PG, + ), + ).toEqual({ reason: "resource-limit", status: "unavailable" }); + + const fromAccessor = { type: "select" }; + Object.defineProperty(fromAccessor, "from", { + get() { + throw new Error("private from getter"); + }, + }); + expect(normalize(fromAccessor, "SELECT 1")).toEqual({ + reason: "malformed-ast", + status: "unavailable", + }); + expect( + normalize({ from: null, type: "select" }, "SELECT 1").status, + ).toBe("ready"); + }); + + it("bounds recursive CTEs, unclosed derived sites, and identifier paths", () => { + const recursive = { + type: "select", + with: [] as unknown[], + }; + recursive.with.push({ name: "self", stmt: recursive }); + expect( + normalize( + recursive, + "WITH self AS (SELECT 1) SELECT 1", + ), + ).toEqual({ reason: "malformed-ast", status: "unavailable" }); + + const child = { type: "select" }; + expect( + normalize( + { + from: [{ as: "d", expr: { ast: child } }], + type: "select", + }, + "SELECT * FROM (SELECT 1", + ), + ).toMatchObject({ + model: { coverage: { relationBindings: "partial" } }, + status: "ready", + }); + + const longName = "x".repeat(257); + expect( + normalize( + { from: [relation(longName)], type: "select" }, + `SELECT * FROM ${longName}`, + ), + ).toMatchObject({ + model: { + bindings: [{ source: { kind: "unknown" } }], + coverage: { relationBindings: "partial" }, + }, + status: "ready", + }); + + const manyComponents = Array.from( + { length: 9 }, + (_, index) => `p${index}`, + ).join("."); + expect( + normalize( + { from: [relation(manyComponents)], type: "select" }, + `SELECT * FROM \`${manyComponents}\``, + BQ, + ), + ).toMatchObject({ + model: { bindings: [{ source: { kind: "unknown" } }] }, + status: "ready", + }); + }); + + it("covers alternate bounded grammar shapes without widening evidence", () => { + expect(normalize({ type: "union" }, "SELECT 1").status).toBe("ready"); + expect( + normalize( + { + from: [{ as: null, schema: "public", table: "users" }], + type: "select", + }, + "SELECT * FROM public.users", + ), + ).toMatchObject({ + model: { + bindings: [ + { + source: { + path: [{ value: "public" }, { value: "users" }], + }, + }, + ], + }, + status: "ready", + }); + expect( + normalize( + { + type: "select", + with: [{ name: "x", stmt: { type: "select" } }], + }, + "WITH x AS (SELECT 1) SELECT 1", + ).status, + ).toBe("ready"); + expect( + normalize( + { from: [relation("users")], type: "select" }, + 'SELECT * FROM ""', + ).status, + ).toBe("ready"); + expect(normalize({ type: "select" }, "SELECT 1 ON true").status).toBe( + "ready", + ); + expect( + normalize( + { type: "select" }, + `SELECT $${"a".repeat(300)}$`, + ), + ).toEqual({ reason: "resource-limit", status: "unavailable" }); + expect( + normalizeNodeSqlParserQueryBindings( + { type: "select" }, + 1, + {}, + PG, + ), + ).toEqual({ reason: "resource-limit", status: "unavailable" }); + }); + + it("rejects malformed roots, syntax shapes, and resource excess", () => { + expect(normalize(null, "SELECT 1")).toEqual({ + reason: "not-query", + status: "unavailable", + }); + expect(normalize({ type: "insert" }, "INSERT INTO x VALUES (1)")).toEqual({ + reason: "not-query", + status: "unavailable", + }); + expect(normalize({ type: "select" }, "x".repeat(16 * 1024 + 1))).toEqual({ + reason: "resource-limit", + status: "unavailable", + }); + expect(normalize({ type: "select" }, "SELECT 1)")).toEqual({ + reason: "resource-limit", + status: "unavailable", + }); + expect(normalize({ type: "select" }, "not a query")).toEqual({ + reason: "unsupported-shape", + status: "unavailable", + }); + }); + + it("rejects hostile wire binding payloads without invoking getters", () => { + let calls = 0; + const hostile = {}; + Object.defineProperty(hostile, "statementRange", { + enumerable: true, + get() { + calls += 1; + return { from: 0, to: 8 }; + }, + }); + const decoded = decodeNodeSqlParserWireMessage({ + kind: "parsed", + protocolVersion: 2, + queryBindings: hostile, + requestId: 1, + statementKind: "query", + }); + expect(decoded).toBeNull(); + expect(calls).toBe(0); + }); +}); diff --git a/src/vnext/__tests__/node-sql-parser-wire.test.ts b/src/vnext/__tests__/node-sql-parser-wire.test.ts index 7647ca5..11a5eb3 100644 --- a/src/vnext/__tests__/node-sql-parser-wire.test.ts +++ b/src/vnext/__tests__/node-sql-parser-wire.test.ts @@ -47,6 +47,7 @@ const validMessages: readonly NodeSqlParserWireMessage[] = [ ...statementKinds.map( (statementKind): NodeSqlParserWireMessage => ({ kind: "parsed", + queryBindings: null, protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, requestId: 1, statementKind, @@ -207,7 +208,7 @@ describe("node-sql-parser wire request codec", () => { expect(encoded).toStrictEqual({ grammar, kind: "parse", - protocolVersion: 1, + protocolVersion: 2, requestId: Number.MAX_SAFE_INTEGER, text: " SELECT 1\n", }); @@ -289,7 +290,7 @@ describe("node-sql-parser wire request codec", () => { ).toBeNull(); }); - it.each([0, 2, "1", null])( + it.each([0, 1, 3, "2", null])( "rejects protocol version %#", (protocolVersion) => { expect( @@ -376,7 +377,8 @@ describe("node-sql-parser wire message codec", () => { encodeNodeSqlParserWireBackendOutcome(7, outcome), ).toStrictEqual({ kind: "parsed", - protocolVersion: 1, + queryBindings: null, + protocolVersion: 2, requestId: 7, statementKind, }); @@ -392,7 +394,7 @@ describe("node-sql-parser wire message codec", () => { { kind: "syntax-rejected" }, { kind: "syntax-rejected", - protocolVersion: 1, + protocolVersion: 2, requestId: 3, }, ], @@ -400,7 +402,7 @@ describe("node-sql-parser wire message codec", () => { { kind: "unsupported", reason: "multiple-statements" }, { kind: "unsupported", - protocolVersion: 1, + protocolVersion: 2, reason: "multiple-statements", requestId: 3, }, @@ -409,7 +411,7 @@ describe("node-sql-parser wire message codec", () => { { kind: "unsupported", reason: "resource-limit" }, { kind: "unsupported", - protocolVersion: 1, + protocolVersion: 2, reason: "resource-limit", requestId: 3, }, @@ -419,7 +421,7 @@ describe("node-sql-parser wire message codec", () => { { code: "backend", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 3, }, ], @@ -432,7 +434,7 @@ describe("node-sql-parser wire message codec", () => { { code: "malformed-output", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 3, }, ], @@ -441,7 +443,7 @@ describe("node-sql-parser wire message codec", () => { { code: "module-load", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 3, }, ], @@ -450,7 +452,7 @@ describe("node-sql-parser wire message codec", () => { const ready = encodeNodeSqlParserWireReady(); expect(ready).toStrictEqual({ kind: "ready", - protocolVersion: 1, + protocolVersion: 2, }); expect(Object.isFrozen(ready)).toBe(true); for (const [outcome, expected] of cases) { @@ -493,6 +495,7 @@ describe("node-sql-parser wire message codec", () => { expect(Reflect.ownKeys(parsed)).toStrictEqual([ "kind", "protocolVersion", + "queryBindings", "requestId", "statementKind", ]); @@ -532,7 +535,7 @@ describe("node-sql-parser wire message codec", () => { expect(protocolError).toStrictEqual({ code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, }); expect(Reflect.ownKeys(protocolError)).not.toContain("requestId"); expect(Object.isFrozen(protocolError)).toBe(true); @@ -568,7 +571,7 @@ describe("node-sql-parser wire message codec", () => { expect( decodeNodeSqlParserWireMessage({ kind: "syntax-rejected", - protocolVersion: 1, + protocolVersion: 2, requestId, }), ).toBeNull(); @@ -584,7 +587,8 @@ describe("node-sql-parser wire message codec", () => { expect( decodeNodeSqlParserWireMessage({ kind: "parsed", - protocolVersion: 1, + queryBindings: null, + protocolVersion: 2, requestId: 1, statementKind, }), @@ -600,7 +604,7 @@ describe("node-sql-parser wire message codec", () => { expect( decodeNodeSqlParserWireMessage({ kind: "unsupported", - protocolVersion: 1, + protocolVersion: 2, reason, requestId: 1, }), @@ -617,7 +621,7 @@ describe("node-sql-parser wire message codec", () => { decodeNodeSqlParserWireMessage({ code, kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 1, }), ).toBeNull(); @@ -632,12 +636,12 @@ describe("node-sql-parser wire message codec", () => { expect( decodeNodeSqlParserWireMessage({ kind, - protocolVersion: 1, + protocolVersion: 2, }), ).toBeNull(); }); - it.each([0, 2, "1", null])( + it.each([0, 1, 3, "2", null])( "rejects response protocol version %#", (protocolVersion) => { expect( @@ -654,7 +658,7 @@ describe("node-sql-parser wire message codec", () => { decodeNodeSqlParserWireMessage({ code: "module-load", kind: "failed", - protocolVersion: 1, + protocolVersion: 2, requestId: 1, retryable: true, }), @@ -663,7 +667,7 @@ describe("node-sql-parser wire message codec", () => { decodeNodeSqlParserWireMessage({ code: "invalid-request", kind: "protocol-error", - protocolVersion: 1, + protocolVersion: 2, requestId: 1, }), ).toBeNull(); diff --git a/src/vnext/__tests__/query-binding-model.bench.ts b/src/vnext/__tests__/query-binding-model.bench.ts new file mode 100644 index 0000000..ae742fa --- /dev/null +++ b/src/vnext/__tests__/query-binding-model.bench.ts @@ -0,0 +1,157 @@ +import { bench, describe } from "vitest"; +import { + createSqlQueryBindingModel, + MAX_QUERY_BINDING_BLOCKS, + MAX_QUERY_BINDINGS, + MAX_QUERY_BINDING_SCOPES, + MAX_QUERY_VISIBILITY_REGIONS, + resolveSqlRelationQualifier, + visibleSqlRelationBindingsAt, +} from "../query-binding-model.js"; +import type { SqlIdentifierComponent } from "../types.js"; + +const BINDING_COUNT = 1_000; +const TEXT = "x".repeat(10 * 1_024); +const AUTHORITY = {}; + +function component(value: string): SqlIdentifierComponent { + return { quoted: false, value }; +} + +const bindings = Array.from({ length: BINDING_COUNT }, (_, index) => ({ + alias: null, + owner: 0, + range: { from: index, to: index + 1 }, + source: { + kind: "named", + path: [component(`relation_${index}`)], + }, +})); + +const scopes = [ + { addedBinding: null, parentScope: null }, + ...bindings.map((_binding, index) => ({ + addedBinding: index, + parentScope: index, + })), +]; + +const input = { + bindings, + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: { from: 0, to: TEXT.length }, + }, + ], + coverage: { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }, + issues: [], + regions: [ + { + block: 0, + kind: "select-list", + range: { from: 0, to: TEXT.length }, + scope: BINDING_COUNT, + }, + ], + scopes, + statementRange: { from: 0, to: TEXT.length }, +}; + +const model = createSqlQueryBindingModel(TEXT, AUTHORITY, input); + +const MAXIMUM_TEXT = "x".repeat(MAX_QUERY_VISIBILITY_REGIONS * 2); +const maximumBlocks = Array.from( + { length: MAX_QUERY_BINDING_BLOCKS }, + (_, index) => ({ + baseScope: 0, + kind: "select", + parentBlock: index === 0 ? null : index - 1, + range: { from: 0, to: MAXIMUM_TEXT.length }, + }), +); +const maximumBindings = Array.from( + { length: MAX_QUERY_BINDINGS }, + (_, index) => ({ + alias: null, + owner: index % MAX_QUERY_BINDING_BLOCKS, + range: { from: 0, to: 1 }, + source: { + kind: "named", + path: [component(`relation_${index}`)], + }, + }), +); +const maximumScopes: { + addedBinding: number | null; + parentScope: number | null; +}[] = [ + { addedBinding: null, parentScope: null }, + ...maximumBindings.map((_binding, index) => ({ + addedBinding: index, + parentScope: index, + })), +]; +while (maximumScopes.length < MAX_QUERY_BINDING_SCOPES) { + maximumScopes.push({ + addedBinding: null, + parentScope: maximumScopes.length - 1, + }); +} +const maximumInput = { + bindings: maximumBindings, + blocks: maximumBlocks, + coverage: { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }, + issues: [], + regions: Array.from( + { length: MAX_QUERY_VISIBILITY_REGIONS }, + (_, index) => ({ + block: MAX_QUERY_BINDING_BLOCKS - 1, + kind: "where", + range: { from: index * 2, to: index * 2 + 1 }, + scope: MAX_QUERY_BINDING_SCOPES - 1, + }), + ), + scopes: maximumScopes, + statementRange: { from: 0, to: MAXIMUM_TEXT.length }, +}; + +function asciiEqual( + left: SqlIdentifierComponent, + right: SqlIdentifierComponent, +): boolean { + return left.value === right.value; +} + +describe("query binding model", () => { + bench("validate 1,000 bindings in a 10 KiB statement", () => { + createSqlQueryBindingModel(TEXT, AUTHORITY, input); + }); + + bench("validate maximum relationship dimensions", () => { + createSqlQueryBindingModel(MAXIMUM_TEXT, AUTHORITY, maximumInput); + }); + + bench("walk 1,000 visible bindings", () => { + visibleSqlRelationBindingsAt(model, 5_000); + }); + + bench("resolve the last of 1,000 qualifiers", () => { + resolveSqlRelationQualifier( + model, + 5_000, + component("relation_999"), + asciiEqual, + ); + }); +}); diff --git a/src/vnext/__tests__/query-binding-model.test.ts b/src/vnext/__tests__/query-binding-model.test.ts new file mode 100644 index 0000000..7126094 --- /dev/null +++ b/src/vnext/__tests__/query-binding-model.test.ts @@ -0,0 +1,1176 @@ +import { describe, expect, it } from "vitest"; +import { + createSqlQueryBindingModel, + isSqlQueryBindingModel, + isSqlQueryBindingModelError, + MAX_QUERY_BINDING_BLOCKS, + MAX_QUERY_BINDINGS, + MAX_QUERY_BINDING_SCOPES, + MAX_QUERY_VISIBILITY_REGIONS, + resolveSqlRelationQualifier, + sqlQueryBindingModelMatches, + visibleSqlRelationBindingsAt, + type SqlQueryBindingModel, + type SqlQueryBindingModelErrorCode, +} from "../query-binding-model.js"; +import type { SqlIdentifierComponent } from "../types.js"; + +const SQL = + "SELECT a.id FROM users AS a JOIN teams t ON a.team_id=t.id WHERE a.id>0"; + +function component(value: string, quoted = false) { + return { quoted, value }; +} + +function range(from: number, to: number) { + return { from, to }; +} + +function completeCoverage() { + return { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "complete", + }; +} + +function fixture(overrides: Readonly> = {}) { + return { + bindings: [ + { + alias: { + explicit: true, + name: component("a"), + range: range(26, 27), + }, + owner: 0, + range: range(17, 27), + source: { + kind: "named", + path: [component("users")], + }, + }, + { + alias: { + explicit: false, + name: component("t"), + range: range(39, 40), + }, + owner: 0, + range: range(33, 40), + source: { + kind: "named", + path: [component("public"), component("teams")], + }, + }, + ], + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(0, SQL.length), + }, + ], + coverage: completeCoverage(), + issues: [], + regions: [ + { + block: 0, + kind: "select-list", + range: range(7, 11), + scope: 2, + }, + { + block: 0, + kind: "join-condition", + range: range(44, 59), + scope: 2, + }, + { + block: 0, + kind: "where", + range: range(66, SQL.length), + scope: 2, + }, + ], + scopes: [ + { addedBinding: null, parentScope: null }, + { addedBinding: 0, parentScope: 0 }, + { addedBinding: 1, parentScope: 1 }, + ], + statementRange: range(0, SQL.length), + ...overrides, + }; +} + +function model( + overrides: Readonly> = {}, + authority: object = {}, +): SqlQueryBindingModel { + return createSqlQueryBindingModel(SQL, authority, fixture(overrides)); +} + +function asciiEqual( + left: SqlIdentifierComponent, + right: SqlIdentifierComponent, +): boolean { + if (left.quoted || right.quoted) { + return left.quoted === right.quoted && left.value === right.value; + } + return left.value.toLowerCase() === right.value.toLowerCase(); +} + +function expectError( + operation: () => unknown, + code: SqlQueryBindingModelErrorCode, +): void { + try { + operation(); + } catch (error) { + expect(isSqlQueryBindingModelError(error)).toBe(true); + if (isSqlQueryBindingModelError(error)) { + expect(error.code).toBe(code); + } + return; + } + throw new Error("Expected a query binding model error"); +} + +describe("query binding model authentication", () => { + it("creates a deeply immutable model bound to exact source and authority", () => { + const authority = {}; + const input = fixture(); + const result = createSqlQueryBindingModel(SQL, authority, input); + + expect(isSqlQueryBindingModel(result)).toBe(true); + expect(isSqlQueryBindingModel(input)).toBe(false); + expect(sqlQueryBindingModelMatches(result, SQL, authority)).toBe(true); + expect(sqlQueryBindingModelMatches(result, `${SQL} `, authority)).toBe(false); + expect(sqlQueryBindingModelMatches(result, SQL, {})).toBe(false); + expect(Object.isFrozen(result)).toBe(true); + expect(Object.isFrozen(result.blocks)).toBe(true); + expect(Object.isFrozen(result.blocks[0])).toBe(true); + expect(Object.isFrozen(result.bindings[0]?.source)).toBe(true); + expect(Object.isFrozen(result.bindings[0]?.alias?.name)).toBe(true); + }); + + it("does not authenticate structural imitations", () => { + const imitation = fixture(); + expect(isSqlQueryBindingModel(null)).toBe(false); + expect(isSqlQueryBindingModel(1)).toBe(false); + expect(isSqlQueryBindingModel(imitation)).toBe(false); + expect( + sqlQueryBindingModelMatches(imitation, SQL, {}), + ).toBe(false); + expect(isSqlQueryBindingModelError(new Error("no"))).toBe(false); + expect(sqlQueryBindingModelMatches(null, SQL, {})).toBe(false); + }); +}); + +describe("query binding visibility", () => { + it("walks persistent scopes in declaration order", () => { + const result = visibleSqlRelationBindingsAt(model(), 8); + expect(result.status).toBe("ready"); + if (result.status === "ready") { + expect(result.region.kind).toBe("select-list"); + expect( + result.bindings.map((binding) => binding.alias?.name.value), + ).toEqual(["a", "t"]); + expect(result.coverage).toBe("complete"); + expect(result.issues).toEqual([]); + } + }); + + it("uses half-open regions and rejects invalid requests", () => { + const authenticated = model(); + expect(visibleSqlRelationBindingsAt(authenticated, 11)).toEqual({ + reason: "outside-visibility-region", + status: "unavailable", + }); + expect(visibleSqlRelationBindingsAt(authenticated, 44).status).toBe("ready"); + expect(visibleSqlRelationBindingsAt(authenticated, SQL.length)).toEqual({ + reason: "outside-visibility-region", + status: "unavailable", + }); + expect(visibleSqlRelationBindingsAt(authenticated, -1)).toEqual({ + reason: "invalid-model", + status: "unavailable", + }); + expect(visibleSqlRelationBindingsAt(authenticated, 1.5)).toEqual({ + reason: "invalid-model", + status: "unavailable", + }); + expect( + visibleSqlRelationBindingsAt(fixture(), 8), + ).toEqual({ reason: "invalid-model", status: "unavailable" }); + }); + + it("combines relation and visibility coverage", () => { + for (const coverage of [ + { + queryBlocks: "partial", + relationBindings: "complete", + visibility: "complete", + }, + { + queryBlocks: "complete", + relationBindings: "partial", + visibility: "complete", + }, + { + queryBlocks: "complete", + relationBindings: "complete", + visibility: "partial", + }, + ]) { + const result = visibleSqlRelationBindingsAt(model({ coverage }), 8); + expect(result.status === "ready" && result.coverage).toBe( + coverage.queryBlocks === "partial" ? "complete" : "partial", + ); + } + }); +}); + +describe("query binding qualifier resolution", () => { + it("resolves aliases and makes an alias hide the base name", () => { + const authenticated = model(); + const alias = resolveSqlRelationQualifier( + authenticated, + 8, + component("A"), + asciiEqual, + ); + expect(alias.status).toBe("resolved"); + if (alias.status === "resolved") { + expect(alias.binding.alias?.name.value).toBe("a"); + expect(alias.coverage).toBe("complete"); + } + expect( + resolveSqlRelationQualifier( + authenticated, + 8, + component("users"), + asciiEqual, + ), + ).toEqual({ status: "no-match" }); + expect( + resolveSqlRelationQualifier( + authenticated, + 8, + component("TEAMS"), + asciiEqual, + ).status, + ).toBe("no-match"); + }); + + it("uses the last named path component and a CTE name without aliases", () => { + const bindings = [ + { + alias: null, + owner: 0, + range: range(17, 22), + source: { + kind: "named", + path: [component("public"), component("users")], + }, + }, + { + alias: null, + owner: 0, + range: range(33, 40), + source: { + declarationRange: range(0, 4), + kind: "cte", + name: component("teams"), + }, + }, + ]; + const authenticated = model({ bindings }); + expect( + resolveSqlRelationQualifier( + authenticated, + 8, + component("users"), + asciiEqual, + ).status, + ).toBe("resolved"); + expect( + resolveSqlRelationQualifier( + authenticated, + 8, + component("teams"), + asciiEqual, + ).status, + ).toBe("resolved"); + }); + + it("reports ambiguity and partial evidence explicitly", () => { + const bindings = [ + fixture().bindings[0], + { + ...fixture().bindings[1], + alias: { + explicit: false, + name: component("a"), + range: range(39, 40), + }, + }, + ]; + const ambiguous = resolveSqlRelationQualifier( + model({ bindings }), + 8, + component("a"), + asciiEqual, + ); + expect(ambiguous.status).toBe("ambiguous"); + if (ambiguous.status === "ambiguous") { + expect(ambiguous.bindings).toHaveLength(2); + expect(Object.isFrozen(ambiguous.bindings)).toBe(true); + } + + const coverage = { + queryBlocks: "complete", + relationBindings: "partial", + visibility: "complete", + }; + const partial = model({ coverage }); + const found = resolveSqlRelationQualifier( + partial, + 8, + component("a"), + asciiEqual, + ); + expect(found.status === "resolved" && found.coverage).toBe("partial"); + expect( + resolveSqlRelationQualifier( + partial, + 8, + component("missing"), + asciiEqual, + ), + ).toEqual({ reason: "partial-coverage", status: "unavailable" }); + }); + + it("propagates unavailable visibility and ignores unqualified derived sources", () => { + const blocks = [ + fixture().blocks[0], + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(17, 22), + }, + ]; + const bindings = [ + { + alias: null, + owner: 0, + range: range(17, 22), + source: { block: 1, kind: "derived" }, + }, + fixture().bindings[1], + ]; + const authenticated = model({ bindings, blocks }); + expect( + resolveSqlRelationQualifier( + authenticated, + 12, + component("t"), + asciiEqual, + ), + ).toEqual({ + reason: "outside-visibility-region", + status: "unavailable", + }); + expect( + resolveSqlRelationQualifier( + authenticated, + 8, + component("anything"), + asciiEqual, + ), + ).toEqual({ status: "no-match" }); + }); +}); + +describe("query binding model validation", () => { + it("rejects invalid source and authority inputs", () => { + expectError( + () => createSqlQueryBindingModel(1, {}, fixture()), + "invalid-source", + ); + expectError( + () => createSqlQueryBindingModel("", {}, fixture()), + "invalid-source", + ); + expectError( + () => + createSqlQueryBindingModel( + "x".repeat(16 * 1024 + 1), + {}, + fixture(), + ), + "resource-limit", + ); + expectError( + () => + createSqlQueryBindingModel( + SQL, + null, + fixture(), + ), + "invalid-authority", + ); + }); + + it.each([ + ["null input", null], + ["missing fields", {}], + ["no blocks", fixture({ blocks: [] })], + ["no scopes", fixture({ scopes: [] })], + [ + "wrong statement extent", + fixture({ statementRange: range(0, SQL.length - 1) }), + ], + [ + "empty statement extent", + fixture({ statementRange: range(0, 0) }), + ], + [ + "unsupported coverage", + fixture({ + coverage: { + ...completeCoverage(), + visibility: "unknown", + }, + }), + ], + ["non-array blocks", fixture({ blocks: {} })], + ])("rejects %s", (_name, input) => { + expectError( + () => createSqlQueryBindingModel(SQL, {}, input), + "invalid-model", + ); + }); + + it("rejects resource excess and sparse arrays", () => { + expectError( + () => + createSqlQueryBindingModel( + SQL, + {}, + fixture({ + blocks: Array.from( + { length: MAX_QUERY_BINDING_BLOCKS + 1 }, + () => fixture().blocks[0], + ), + }), + ), + "resource-limit", + ); + const sparse: unknown[] = []; + sparse.length = 1; + expectError( + () => + createSqlQueryBindingModel( + SQL, + {}, + fixture({ blocks: sparse }), + ), + "invalid-model", + ); + }); + + it("never invokes accessors", () => { + let calls = 0; + const hostile = fixture(); + Object.defineProperty(hostile, "blocks", { + enumerable: true, + get() { + calls += 1; + return []; + }, + }); + expectError( + () => createSqlQueryBindingModel(SQL, {}, hostile), + "invalid-model", + ); + expect(calls).toBe(0); + }); + + it("rejects malformed primitive fields without coercion", () => { + const invalidCases: readonly Readonly>[] = [ + { + bindings: [ + { + ...fixture().bindings[0], + alias: { + ...fixture().bindings[0]?.alias, + explicit: "yes", + }, + }, + fixture().bindings[1], + ], + }, + { + blocks: [ + { ...fixture().blocks[0], kind: 1 }, + ], + }, + { + blocks: [ + { ...fixture().blocks[0], baseScope: -1 }, + ], + }, + { + blocks: [ + { ...fixture().blocks[0], baseScope: 0.5 }, + ], + }, + { + regions: [ + { + ...fixture().regions[0], + range: range(11, 7), + }, + ], + }, + { + regions: [ + { + ...fixture().regions[0], + range: range(7, 7), + }, + ], + }, + ]; + for (const invalidCase of invalidCases) { + expectError(() => model(invalidCase), "invalid-model"); + } + }); + + it("rejects arrays whose length cannot be inspected as data", () => { + const hostileArray = new Proxy([], { + getOwnPropertyDescriptor(target, key) { + if (key === "length") { + return { + configurable: false, + enumerable: false, + value: "one", + writable: true, + }; + } + return Reflect.getOwnPropertyDescriptor(target, key); + }, + }); + expectError( + () => model({ blocks: hostileArray }), + "invalid-model", + ); + }); + + it("normalizes proxy and inspection failures", () => { + const hostile = new Proxy(fixture(), { + getOwnPropertyDescriptor() { + throw new Error("hostile trap"); + }, + }); + expectError( + () => createSqlQueryBindingModel(SQL, {}, hostile), + "invalid-model", + ); + }); + + it.each([ + [ + "forward block parent", + { + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: 1, + range: range(0, SQL.length), + }, + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(0, SQL.length), + }, + ], + }, + ], + [ + "child outside parent", + { + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(0, 20), + }, + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(19, 30), + }, + ], + }, + ], + [ + "forward scope parent", + { + scopes: [ + { addedBinding: null, parentScope: 1 }, + { addedBinding: null, parentScope: null }, + ], + }, + ], + [ + "repeated binding in scope chain", + { + scopes: [ + { addedBinding: 0, parentScope: null }, + { addedBinding: 0, parentScope: 0 }, + { addedBinding: 1, parentScope: 1 }, + ], + }, + ], + [ + "overlapping visibility", + { + regions: [ + { + block: 0, + kind: "select-list", + range: range(7, 20), + scope: 2, + }, + { + block: 0, + kind: "where", + range: range(19, 22), + scope: 2, + }, + ], + }, + ], + [ + "unordered visibility", + { + regions: [ + { + block: 0, + kind: "where", + range: range(66, 70), + scope: 2, + }, + { + block: 0, + kind: "select-list", + range: range(7, 11), + scope: 2, + }, + ], + }, + ], + ])("rejects %s", (_name, overrides) => { + expectError( + () => model(overrides), + "invalid-model", + ); + }); + + it("rejects invalid binding ownership and source relationships", () => { + const outside = [ + { + ...fixture().bindings[0], + range: range(0, SQL.length), + }, + fixture().bindings[1], + ]; + const narrowBlock = [ + { + ...fixture().blocks[0], + range: range(5, SQL.length), + }, + ]; + expectError(() => model({ bindings: outside, blocks: narrowBlock }), "invalid-model"); + + const blocks = [ + fixture().blocks[0], + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(17, 22), + }, + ]; + const bindings = [ + { + ...fixture().bindings[0], + source: { block: 1, kind: "derived" }, + }, + fixture().bindings[1], + ]; + expectError(() => model({ bindings, blocks }), "invalid-model"); + }); + + it("rejects a derived binding that points at a sibling outside its range", () => { + const text = "x".repeat(20); + expectError( + () => + createSqlQueryBindingModel(text, {}, { + bindings: [ + { + alias: null, + owner: 0, + range: range(1, 5), + source: { block: 2, kind: "derived" }, + }, + ], + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(0, 20), + }, + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(1, 5), + }, + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(10, 15), + }, + ], + coverage: completeCoverage(), + issues: [], + regions: [], + scopes: [ + { addedBinding: null, parentScope: null }, + { addedBinding: 0, parentScope: 0 }, + ], + statementRange: range(0, text.length), + }), + "invalid-model", + ); + }); + + it("bounds maximum relationship validation work", () => { + const text = "x".repeat(MAX_QUERY_VISIBILITY_REGIONS * 2); + const blocks = Array.from( + { length: MAX_QUERY_BINDING_BLOCKS }, + (_, index) => ({ + baseScope: 0, + kind: "select", + parentBlock: index === 0 ? null : index - 1, + range: range(0, text.length), + }), + ); + const bindings = Array.from( + { length: MAX_QUERY_BINDINGS }, + (_, index) => ({ + alias: null, + owner: index % MAX_QUERY_BINDING_BLOCKS, + range: range(0, 1), + source: { + kind: "named", + path: [component(`r${index}`)], + }, + }), + ); + const scopes: { + addedBinding: number | null; + parentScope: number | null; + }[] = [ + { addedBinding: null, parentScope: null }, + ...bindings.map((_binding, index) => ({ + addedBinding: index, + parentScope: index, + })), + ]; + while (scopes.length < MAX_QUERY_BINDING_SCOPES) { + scopes.push({ + addedBinding: null, + parentScope: scopes.length - 1, + }); + } + const regions = Array.from( + { length: MAX_QUERY_VISIBILITY_REGIONS }, + (_, index) => ({ + block: MAX_QUERY_BINDING_BLOCKS - 1, + kind: "where", + range: range(index * 2, index * 2 + 1), + scope: MAX_QUERY_BINDING_SCOPES - 1, + }), + ); + const start = performance.now(); + const result = createSqlQueryBindingModel(text, {}, { + bindings, + blocks, + coverage: completeCoverage(), + issues: [], + regions, + scopes, + statementRange: range(0, text.length), + }); + const duration = performance.now() - start; + + expect(result.bindings).toHaveLength(MAX_QUERY_BINDINGS); + expect(result.regions).toHaveLength(MAX_QUERY_VISIBILITY_REGIONS); + expect(duration).toBeLessThan(500); + }); + + it("requires partial relation coverage for unknown sources", () => { + const bindings = [ + { + ...fixture().bindings[0], + source: { + kind: "unknown", + reason: "unsupported-relation-source", + }, + }, + fixture().bindings[1], + ]; + expectError(() => model({ bindings }), "invalid-model"); + const coverage = { + ...completeCoverage(), + relationBindings: "partial", + }; + expect(model({ bindings, coverage }).bindings[0]?.source.kind).toBe( + "unknown", + ); + }); + + it("rejects malformed identifiers, paths, ranges, aliases, enums, and indexes", () => { + const cases: readonly Readonly>[] = [ + { + bindings: [ + { + ...fixture().bindings[0], + source: { kind: "named", path: [] }, + }, + fixture().bindings[1], + ], + }, + { + bindings: [ + { + ...fixture().bindings[0], + source: { + kind: "named", + path: [component("\ud800")], + }, + }, + fixture().bindings[1], + ], + }, + { + bindings: [ + { + ...fixture().bindings[0], + alias: { + explicit: true, + name: component("a"), + range: range(28, 29), + }, + }, + fixture().bindings[1], + ], + }, + { + bindings: [ + { ...fixture().bindings[0], owner: 99 }, + fixture().bindings[1], + ], + }, + { + bindings: [ + { + ...fixture().bindings[0], + source: { kind: "mystery" }, + }, + fixture().bindings[1], + ], + }, + { + blocks: [ + { ...fixture().blocks[0], kind: "values" }, + ], + }, + { + regions: [ + { ...fixture().regions[0], kind: "window" }, + ], + }, + { + issues: [{ code: "mystery", range: range(0, 1) }], + }, + { + scopes: [ + { addedBinding: 99, parentScope: null }, + ], + }, + ]; + for (let index = 0; index < cases.length; index += 1) { + const overrides = cases[index]; + if (!overrides) { + throw new Error("Malformed fixture table contains a hole"); + } + expectError(() => model(overrides), "invalid-model"); + } + }); + + it("accepts every closed issue and unknown reason", () => { + const codes = [ + "ambiguous-alias", + "duplicate-alias", + "opaque-template-context", + "parser-compatibility", + "recursive-cte-uncertainty", + "resource-limit", + "unknown-correlation", + "unsupported-clause", + "unsupported-relation-source", + ]; + const coverage = { + queryBlocks: "partial", + relationBindings: "partial", + visibility: "partial", + }; + const issues = codes.map((code, index) => ({ + code, + range: range(index, index + 1), + })); + expect(model({ coverage, issues }).issues).toHaveLength(codes.length); + + for (const reason of [ + "opaque-template-context", + "unknown-correlation", + "unsupported-relation-source", + ]) { + const bindings = [ + { + ...fixture().bindings[0], + source: { kind: "unknown", reason }, + }, + fixture().bindings[1], + ]; + expect(model({ bindings, coverage }).bindings[0]?.source).toEqual({ + kind: "unknown", + reason, + }); + } + }); + + it("accepts well-formed surrogate pairs and rejects lone trailing surrogates", () => { + const validBindings = [ + { + ...fixture().bindings[0], + source: { + kind: "named", + path: [component("relation_\ud83d\ude80")], + }, + }, + fixture().bindings[1], + ]; + expect(model({ bindings: validBindings }).bindings[0]?.source.kind).toBe( + "named", + ); + const invalidBindings = [ + { + ...fixture().bindings[0], + source: { + kind: "named", + path: [component("\udc00")], + }, + }, + fixture().bindings[1], + ]; + expectError(() => model({ bindings: invalidBindings }), "invalid-model"); + }); + + it("rejects a region assigned outside its query block", () => { + const blocks = [ + fixture().blocks[0], + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(17, 22), + }, + ]; + const regions = [ + { + block: 1, + kind: "select-list", + range: range(7, 11), + scope: 2, + }, + ]; + expectError(() => model({ blocks, regions }), "invalid-model"); + }); + + it("allows parent correlation but rejects local or sibling bindings in entry scopes", () => { + const correlatedBlocks = [ + fixture().blocks[0], + { + baseScope: 1, + kind: "select", + parentBlock: 0, + range: range(44, 59), + }, + ]; + const correlatedRegions = [ + { + block: 1, + kind: "where", + range: range(45, 50), + scope: 1, + }, + ]; + expect( + model({ + blocks: correlatedBlocks, + regions: correlatedRegions, + }).blocks[1]?.baseScope, + ).toBe(1); + + const localEntryBlocks = [ + { ...fixture().blocks[0], baseScope: 1 }, + ]; + expectError( + () => model({ blocks: localEntryBlocks }), + "invalid-model", + ); + + const siblingBlocks = [ + fixture().blocks[0], + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(17, 22), + }, + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(33, 40), + }, + ]; + const siblingBindings = [ + { + alias: null, + owner: 1, + range: range(17, 22), + source: { kind: "named", path: [component("left_child")] }, + }, + { + alias: null, + owner: 0, + range: range(33, 40), + source: { kind: "named", path: [component("root_relation")] }, + }, + ]; + const siblingScopes = [ + { addedBinding: null, parentScope: null }, + { addedBinding: 0, parentScope: 0 }, + { addedBinding: 1, parentScope: 0 }, + ]; + const siblingRegions = [ + { + block: 2, + kind: "select-list", + range: range(34, 38), + scope: 1, + }, + ]; + expectError( + () => + model({ + bindings: siblingBindings, + blocks: siblingBlocks, + regions: siblingRegions, + scopes: siblingScopes, + }), + "invalid-model", + ); + }); + + it("rejects overlapping sibling query blocks", () => { + const blocks = [ + fixture().blocks[0], + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(17, 30), + }, + { + baseScope: 0, + kind: "select", + parentBlock: 0, + range: range(20, 40), + }, + ]; + expectError(() => model({ blocks }), "invalid-model"); + }); + + it("preserves invariants across generated scope chains", () => { + for (let count = 1; count <= 32; count += 1) { + const text = "x".repeat(Math.max(count, 2)); + const bindings = Array.from({ length: count }, (_, index) => ({ + alias: null, + owner: 0, + range: range(index, index + 1), + source: { + kind: "named", + path: [component(`r${index}`)], + }, + })); + const scopes = [ + { addedBinding: null, parentScope: null }, + ...bindings.map((_binding, index) => ({ + addedBinding: index, + parentScope: index, + })), + ]; + const generated = createSqlQueryBindingModel(text, {}, { + bindings, + blocks: [ + { + baseScope: 0, + kind: "select", + parentBlock: null, + range: range(0, text.length), + }, + ], + coverage: completeCoverage(), + issues: [], + regions: [ + { + block: 0, + kind: "other", + range: range(0, text.length), + scope: count, + }, + ], + scopes, + statementRange: range(0, text.length), + }); + const visible = visibleSqlRelationBindingsAt(generated, 0); + expect(visible.status === "ready" && visible.bindings).toHaveLength(count); + } + }); +}); diff --git a/src/vnext/__tests__/relation-catalog-search-work.test.ts b/src/vnext/__tests__/relation-catalog-search-work.test.ts index a6914c7..f6dbbff 100644 --- a/src/vnext/__tests__/relation-catalog-search-work.test.ts +++ b/src/vnext/__tests__/relation-catalog-search-work.test.ts @@ -1045,7 +1045,9 @@ describe("catalog search ownership and active-slot lifecycle", () => { it("starts observer-only queued work when an active slot becomes free", async () => { const provider = providerHarness(); - const service = coordinator(provider.captured); + const service = coordinator(provider.captured, { + deadlineScheduler: new ManualDeadlineScheduler(), + }); const active = Array.from( { length: MAX_CATALOG_ACTIVE_SEARCH_WORK }, (_, index) => diff --git a/src/vnext/__tests__/relation-completion.test.ts b/src/vnext/__tests__/relation-completion.test.ts index 8a25702..b33efba 100644 --- a/src/vnext/__tests__/relation-completion.test.ts +++ b/src/vnext/__tests__/relation-completion.test.ts @@ -232,11 +232,16 @@ describe("relation completion composition", () => { ); expect( - value.items.map((item) => [ - item.relationKind, - item.label, - item.edit.insert, - ]), + value.items.map((item) => { + if (item.kind !== "relation") { + throw new Error("Expected a relation completion"); + } + return [ + item.relationKind, + item.label, + item.edit.insert, + ]; + }), ).toEqual([ ["cte", "users", "users"], ["cte", "zed", "zed"], diff --git a/src/vnext/__tests__/session.test.ts b/src/vnext/__tests__/session.test.ts index 14c3ad0..c8d3ead 100644 --- a/src/vnext/__tests__/session.test.ts +++ b/src/vnext/__tests__/session.test.ts @@ -6,6 +6,8 @@ import { duckdbDialect, postgresDialect, SqlSessionError, + type SqlColumnCatalogProvider, + type SqlNamespaceCatalogProvider, } from "../index.js"; import { DefaultSqlLanguageService, @@ -598,7 +600,7 @@ describe("relation completion session integration", () => { }); it("does not correlate catalog invalidation after a soft lease expires", async () => { - vi.useFakeTimers(); + vi.useFakeTimers({ shouldAdvanceTime: true }); let service: | ReturnType> | undefined; @@ -1032,209 +1034,1282 @@ describe("relation completion session integration", () => { service.dispose(); }); - it("stops a stale catalog event after a listener updates the session", () => { - let invalidate: - | ((event: SqlCatalogInvalidation) => void) - | undefined; + it("stops a stale catalog event after a listener updates the session", () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = createSqlLanguageService({ + catalog: catalogProvider( + async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }), + (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + ), + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const events: SqlRevision[] = []; + session.onDidChange((event) => { + events.push(event.revision); + session.update({ + baseRevision: event.revision, + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "remote", + }, + }); + }); + session.onDidChange((event) => { + events.push(event.revision); + }); + + invalidate?.({ + epoch: { generation: 1, token: "changed" }, + }); + + expect(events).toHaveLength(1); + expect(session.revision).not.toBe(events[0]); + service.dispose(); + }); + + it("does not deliver an outer catalog event after nested invalidation", () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = createSqlLanguageService({ + catalog: catalogProvider( + async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }), + (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + ), + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const events: SqlRevision[] = []; + let nested = false; + session.onDidChange((event) => { + events.push(event.revision); + if (!nested) { + nested = true; + invalidate?.({ + epoch: { generation: 2, token: "nested" }, + }); + } + }); + session.onDidChange((event) => { + events.push(event.revision); + }); + + invalidate?.({ + epoch: { generation: 1, token: "outer" }, + }); + + expect(events).toHaveLength(4); + expect(events[0]).toBe(events[1]); + expect(events[1]).not.toBe(events[2]); + expect(events[2]).toBe(events[3]); + expect(session.revision).toBe(events[2]); + service.dispose(); + }); + + it("keeps duplicate listener subscriptions independent", () => { + let invalidate: + | ((event: SqlCatalogInvalidation) => void) + | undefined; + const service = createSqlLanguageService({ + catalog: catalogProvider( + async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "initial" }, + relations: [], + status: "ready", + }), + (_scope, listener) => { + invalidate = listener; + return () => undefined; + }, + ), + dialects: [duckdb], + }); + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const events: string[] = []; + const listener = (event: SqlSessionChangeEvent): void => { + events.push(event.reason); + }; + const first = session.onDidChange(listener); + const second = session.onDidChange(listener); + + first.dispose(); + invalidate?.({ + epoch: { generation: 1, token: "first" }, + }); + expect(events).toEqual(["catalog"]); + + second.dispose(); + invalidate?.({ + epoch: { generation: 2, token: "second" }, + }); + expect(events).toEqual(["catalog"]); + service.dispose(); + }); + + it("validates completion configuration and request input", async () => { + expectSessionError("invalid-service-options", () => { + createSqlLanguageService({ + completion: { catalogResponseBudgetMs: 51 }, + dialects: [duckdb], + }); + }); + const { service, session } = openSession(); + await expect( + session.complete(null as never), + ).rejects.toMatchObject({ + code: "invalid-completion-request", + }); + service.dispose(); + }); + + it("rejects catalog context without a configured provider atomically", () => { + const service = createSqlLanguageService({ + dialects: [duckdb], + }); + expectSessionError("invalid-context", () => { + service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + }); + const session = service.openDocument({ + context: { + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ", + }); + const revision = session.revision; + expectSessionError("invalid-context", () => { + session.update({ + baseRevision: revision, + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + }); + }); + expect(session.revision).toBe(revision); + service.dispose(); + }); +}); + +describe("column completion session integration", () => { + const relationCatalog: SqlRelationCatalogProvider = { + id: "relations", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + status: "ready", + }), + }; + + function serviceWithColumns( + loadColumns: SqlColumnCatalogProvider["loadColumns"], + ) { + return createSqlLanguageService({ + catalog: relationCatalog, + columns: { id: "columns", loadColumns }, + completion: { catalogResponseBudgetMs: 40 }, + dialects: [duckdb], + }); + } + + it("batches and completes a qualified alias through the session", async () => { + const requests: Parameters< + SqlColumnCatalogProvider["loadColumns"] + >[0][] = []; + const service = serviceWithColumns(async (request) => { + requests.push(request); + const relation = request.relations[0]; + if (!relation) throw new Error("Expected one relation"); + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: [{ + columns: [{ + columnEntityId: "users:name", + dataType: "VARCHAR", + identifier: { quoted: false, value: "name" }, + insertText: "name", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: "users", + requestKey: relation.requestKey, + status: "ready", + }], + }; + }); + const text = "SELECT u.na FROM app.users AS u"; + const session = service.openDocument({ + context: { + catalog: { + scope: "connection:1", + searchPath: [[{ quoted: false, value: "app" }]], + }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + + await expect( + session.complete({ + position: text.indexOf("na") + 2, + trigger: { kind: "invoked" }, + }), + ).resolves.toMatchObject({ + sources: [{ + feature: "column-catalog", + outcome: "ready", + providerId: "columns", + }], + status: "ready", + value: { + isIncomplete: false, + items: [{ + dataType: "VARCHAR", + edit: { from: 9, insert: "name", to: 11 }, + kind: "column", + label: "name", + provenance: { + columnEntityId: "users:name", + kind: "column-catalog", + providerId: "columns", + relationEntityId: "users", + scope: "connection:1", + }, + }], + }, + }); + expect(requests).toHaveLength(1); + expect(requests[0]).toMatchObject({ + dialectId: "duckdb", + expectedEpoch: null, + relations: [{ + path: [ + { quoted: false, value: "app" }, + { quoted: false, value: "users" }, + ], + requestKey: "binding:0", + }], + scope: "connection:1", + }); + service.dispose(); + }); + + it("loads all visible relations in one unqualified batch", async () => { + let calls = 0; + const service = serviceWithColumns(async (request) => { + calls += 1; + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: request.relations.map((relation, index) => ({ + columns: [{ + columnEntityId: `relation-${index}:id`, + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: `relation-${index}`, + requestKey: relation.requestKey, + status: "ready", + })), + }; + }); + const text = + "SELECT i FROM users u JOIN orders o ON o.user_id = u.id"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const result = await session.complete({ + position: "SELECT i".length, + trigger: { kind: "invoked" }, + }); + + expect(calls).toBe(1); + expect(result).toMatchObject({ + status: "ready", + value: { + items: [ + { detail: "o", label: "id" }, + { detail: "u", label: "id" }, + ], + }, + }); + service.dispose(); + }); + + it("completes a qualified correlated outer relation", async () => { + const requests: Parameters< + SqlColumnCatalogProvider["loadColumns"] + >[0][] = []; + const service = serviceWithColumns(async (request) => { + requests.push(request); + const relation = request.relations[0]; + if (!relation) throw new Error("Expected the correlated relation"); + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: [{ + columns: [{ + columnEntityId: "users:id", + identifier: { quoted: false, value: "id" }, + insertText: "id", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: "users", + requestKey: relation.requestKey, + status: "ready", + }], + }; + }); + const text = + "SELECT * FROM users u WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.)"; + const position = text.indexOf("u.)") + 2; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:correlated" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + + await expect(session.complete({ + position, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + status: "ready", + value: { + items: [{ label: "id" }], + }, + }); + expect(requests).toHaveLength(1); + expect(requests[0]?.relations).toEqual([{ + path: [{ quoted: false, value: "users" }], + requestKey: "binding:1", + }]); + service.dispose(); + }); + + it("completes LATERAL references without loading later siblings", async () => { + for (const dialect of [postgres, duckdb]) { + for (const expression of ["u.", "na"]) { + const requests: Parameters< + SqlColumnCatalogProvider["loadColumns"] + >[0][] = []; + const service = createSqlLanguageService({ + catalog: relationCatalog, + columns: { + id: "columns", + loadColumns: async (request) => { + requests.push(request); + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: request.relations.map((relation, index) => ({ + columns: [{ + columnEntityId: `column-${index}`, + identifier: { quoted: false, value: "name" }, + insertText: "name", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: `relation-${index}`, + requestKey: relation.requestKey, + status: "ready", + })), + }; + }, + }, + dialects: [dialect], + }); + const text = + `SELECT * FROM users u, LATERAL (SELECT ${expression} FROM orders o) x, secrets s`; + const session = service.openDocument({ + context: { + catalog: { + scope: `connection:lateral:${dialect.id}:${expression}`, + }, + dialect: dialect.id, + engine: "local", + }, + text, + }); + const position = text.indexOf(expression) + expression.length; + + await expect(session.complete({ + position, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ status: "ready" }); + expect(requests).toHaveLength(1); + expect(requests[0]?.relations.map((relation) => + relation.path.at(-1)?.value + )).toEqual(expression === "u." + ? ["users"] + : ["orders", "users"]); + service.dispose(); + } + } + }); + + it("never resolves a visible CTE as a physical relation", async () => { + let calls = 0; + const service = serviceWithColumns(async () => { + calls += 1; + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + }; + }); + const text = + "WITH users AS (SELECT 1 AS local_id) SELECT users. FROM users"; + const position = text.indexOf("users.") + "users.".length; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + + await expect(session.complete({ + position, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + status: "unavailable", + }); + expect(calls).toBe(0); + service.dispose(); + }); + + it("keeps slow providers inside the completion response budget", async () => { + vi.useFakeTimers(); + try { + const service = serviceWithColumns(() => new Promise(() => {})); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const completion = session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + await vi.advanceTimersByTimeAsync(40); + const result = await completion; + expect(result).toMatchObject({ + refreshToken: expect.anything(), + sources: [{ + feature: "column-catalog", + outcome: "loading", + }], + status: "ready", + value: { + isIncomplete: true, + issues: [{ reason: "column-catalog-loading" }], + items: [], + }, + }); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); + + it("charges column analysis against the total response budget", async () => { + vi.useFakeTimers(); + const performanceNow = vi.spyOn(performance, "now") + .mockReturnValueOnce(0) + .mockReturnValue(30); + try { + const service = serviceWithColumns(() => new Promise(() => {})); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:column-total-budget" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + let settled = false; + const completion = session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + void completion.then(() => { + settled = true; + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(9); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + await expect(completion).resolves.toMatchObject({ + status: "ready", + value: { + issues: [{ reason: "column-catalog-loading" }], + }, + }); + service.dispose(); + } finally { + performanceNow.mockRestore(); + vi.useRealTimers(); + } + }); + + it("bounds provider-declared column loading to one automatic retry", async () => { + vi.useFakeTimers(); + try { + let calls = 0; + const service = serviceWithColumns(async (request) => { + calls += 1; + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: request.relations.map((relation) => ({ + requestKey: relation.requestKey, + status: "loading", + })), + }; + }); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:columns-loading" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const events: SqlSessionChangeEvent[] = []; + session.onDidChange((event) => events.push(event)); + const result = await session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + if (result.status !== "ready" || result.refreshToken === null) { + throw new Error("Expected retained column loading"); + } + + await vi.advanceTimersByTimeAsync(100); + expect(events).toMatchObject([{ + reason: "catalog-availability", + refreshToken: result.refreshToken, + }]); + const retry = await session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + expect(retry).toMatchObject({ + refreshToken: null, + sources: [{ + feature: "column-catalog", + outcome: "loading", + }], + }); + await vi.advanceTimersByTimeAsync(500); + expect(calls).toBe(2); + expect(events).toHaveLength(1); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); + + it("emits readiness when a timed-out column batch settles", async () => { + vi.useFakeTimers(); + try { + let resolveBatch: + | ((value: Awaited< + ReturnType + >) => void) + | undefined; + const batch = new Promise< + Awaited> + >((resolve) => { + resolveBatch = resolve; + }); + const service = createSqlLanguageService({ + columns: { id: "columns", loadColumns: () => batch }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:columns-ready" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const events: SqlSessionChangeEvent[] = []; + session.onDidChange((event) => events.push(event)); + const pending = session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + await vi.advanceTimersByTimeAsync(0); + const result = await pending; + if (result.status !== "ready" || result.refreshToken === null) { + throw new Error("Expected retained column loading"); + } + + resolveBatch?.({ + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + }); + await Promise.resolve(); + await Promise.resolve(); + expect(events).toMatchObject([{ + reason: "catalog-availability", + refreshToken: result.refreshToken, + }]); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); + + it("emits readiness when a timed-out column batch rejects", async () => { + vi.useFakeTimers(); + try { + let rejectBatch: ((reason: Error) => void) | undefined; + const batch = new Promise< + Awaited> + >((_resolve, reject) => { + rejectBatch = reject; + }); + const service = createSqlLanguageService({ + columns: { id: "columns", loadColumns: () => batch }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:columns-reject" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const events: SqlSessionChangeEvent[] = []; + session.onDidChange((event) => events.push(event)); + const pending = session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + await vi.advanceTimersByTimeAsync(0); + const result = await pending; + if (result.status !== "ready" || result.refreshToken === null) { + throw new Error("Expected retained column loading"); + } + + rejectBatch?.(new Error("offline")); + await Promise.resolve(); + await Promise.resolve(); + expect(events).toMatchObject([{ + reason: "catalog-availability", + refreshToken: result.refreshToken, + }]); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); + + it("expires timed-out column readiness without a stale event", async () => { + vi.useFakeTimers(); + try { + let resolveBatch: + | ((value: Awaited< + ReturnType + >) => void) + | undefined; + const batch = new Promise< + Awaited> + >((resolve) => { + resolveBatch = resolve; + }); + const service = createSqlLanguageService({ + columns: { id: "columns", loadColumns: () => batch }, + completion: { catalogResponseBudgetMs: 0 }, + dialects: [duckdb], + }); + const text = "SELECT u. FROM users u"; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:columns-expire" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const events: SqlSessionChangeEvent[] = []; + session.onDidChange((event) => events.push(event)); + const completion = session.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + await vi.advanceTimersByTimeAsync(0); + await completion; + await vi.advanceTimersByTimeAsync(1_000); + + resolveBatch?.({ + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + }); + await Promise.resolve(); + await Promise.resolve(); + expect(events).toEqual([]); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("namespace completion session integration", () => { + const relationCatalog: SqlRelationCatalogProvider = { + id: "relations", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + status: "ready", + }), + }; + + function serviceWithNamespaces( + search: SqlNamespaceCatalogProvider["search"], + budget = 40, + ) { + return createSqlLanguageService({ + catalog: relationCatalog, + completion: { catalogResponseBudgetMs: budget }, + dialects: [duckdb], + namespaces: { id: "namespaces", search }, + }); + } + + it("merges namespace containers into relation-site completion", async () => { + const requests: Parameters< + SqlNamespaceCatalogProvider["search"] + >[0][] = []; + const service = serviceWithNamespaces(async (request) => { + requests.push(request); + return { + containers: [{ + canonicalPath: [{ + quoted: false, + role: "schema", + value: "main", + }], + containerEntityId: "schema:main", + detail: "DuckDB schema", + insertText: "main", + matchQuality: "exact", + }], + coverage: "complete", + epoch: { generation: 1, token: "epoch-1" }, + status: "ready", + }; + }); + const text = "SELECT * FROM ma"; + const session = service.openDocument({ + context: { + catalog: { + scope: "connection:1", + searchPath: [[{ quoted: false, value: "main" }]], + }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + + await expect(session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + sources: [ + { feature: "relation-catalog" }, + { + feature: "namespace-catalog", + outcome: "ready", + providerId: "namespaces", + }, + ], + status: "ready", + value: { + isIncomplete: false, + items: [{ + detail: "DuckDB schema", + edit: { from: 14, insert: "main", to: 16 }, + kind: "namespace", + label: "main", + provenance: { + containerEntityId: "schema:main", + kind: "namespace-catalog", + providerId: "namespaces", + scope: "connection:1", + }, + role: "schema", + }], + }, + }); + expect(requests).toHaveLength(1); + expect(requests[0]).toMatchObject({ + dialectId: "duckdb", + expectedEpoch: null, + limit: 128, + prefix: { quoted: false, value: "ma" }, + qualifier: [], + scope: "connection:1", + }); + service.dispose(); + }); + + it("keeps slow namespace providers inside the shared budget", async () => { + vi.useFakeTimers(); + try { + const service = serviceWithNamespaces( + () => new Promise(() => {}), + 0, + ); + const text = "SELECT * FROM "; + const session = service.openDocument({ + context: { + catalog: { scope: "connection:1" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const completion = session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }); + await vi.advanceTimersByTimeAsync(0); + await expect(completion).resolves.toMatchObject({ + sources: [ + { feature: "relation-catalog" }, + { + feature: "namespace-catalog", + outcome: "loading", + }, + ], + status: "ready", + value: { + isIncomplete: true, + issues: [{ reason: "namespace-catalog-loading" }], + }, + }); + service.dispose(); + } finally { + vi.useRealTimers(); + } + }); + + it("supports a namespace-only service and preserves partial evidence", async () => { const service = createSqlLanguageService({ - catalog: catalogProvider( - async () => ({ - coverage: { kind: "complete" }, - epoch: { generation: 0, token: "initial" }, - relations: [], + dialects: [duckdb], + namespaces: { + id: "namespace-only", + search: async () => ({ + containers: [{ + canonicalPath: [{ + quoted: false, + role: "schema", + value: "main", + }], + containerEntityId: "schema:main", + insertText: "main", + matchQuality: "equivalent", + }], + coverage: "partial", + epoch: { generation: 1, token: "epoch-1" }, status: "ready", }), - (_scope, listener) => { - invalidate = listener; - return () => undefined; - }, - ), - dialects: [duckdb], + }, }); + const text = "SELECT * FROM m"; const session = service.openDocument({ context: { - catalog: { scope: "connection:1" }, + catalog: { scope: "connection:namespace-only" }, dialect: "duckdb", engine: "local", }, - text: "SELECT * FROM ", + text, }); - const events: SqlRevision[] = []; - session.onDidChange((event) => { - events.push(event.revision); - session.update({ - baseRevision: event.revision, + + await expect(session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + })).resolves.toMatchObject({ + sources: [{ + coverage: "partial", + feature: "namespace-catalog", + }], + status: "ready", + value: { + isIncomplete: true, + issues: [{ reason: "namespace-catalog-partial" }], + items: [{ kind: "namespace", label: "main" }], + }, + }); + service.dispose(); + }); + + it("polls a provider-declared loading namespace with token correlation", async () => { + vi.useFakeTimers(); + try { + let calls = 0; + const service = serviceWithNamespaces(async () => { + calls += 1; + return { + epoch: { generation: 1, token: "epoch-1" }, + status: "loading", + }; + }); + const text = "SELECT * FROM ma"; + const session = service.openDocument({ context: { - catalog: { scope: "connection:1" }, + catalog: { scope: "connection:namespaces-loading" }, dialect: "duckdb", - engine: "remote", + engine: "local", }, + text, + }); + const events: SqlSessionChangeEvent[] = []; + session.onDidChange((event) => events.push(event)); + const result = await session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }); + if (result.status !== "ready" || result.refreshToken === null) { + throw new Error("Expected retained namespace loading"); + } + expect(result.value).toMatchObject({ + issues: [{ + reason: "namespace-catalog-loading", + remainingIntentLeaseMs: 1_000, + }], }); - }); - session.onDidChange((event) => { - events.push(event.revision); - }); - invalidate?.({ - epoch: { generation: 1, token: "changed" }, - }); + await vi.advanceTimersByTimeAsync(100); + expect(events).toMatchObject([{ + reason: "catalog-availability", + refreshToken: result.refreshToken, + }]); - expect(events).toHaveLength(1); - expect(session.revision).not.toBe(events[0]); - service.dispose(); + const retry = await session.complete({ + position: text.length, + trigger: { kind: "invoked" }, + }); + expect(retry).toMatchObject({ + refreshToken: null, + sources: [ + { feature: "relation-catalog" }, + { + feature: "namespace-catalog", + outcome: "loading", + }, + ], + }); + await vi.advanceTimersByTimeAsync(500); + expect(calls).toBe(2); + expect(events).toHaveLength(1); + service.dispose(); + } finally { + vi.useRealTimers(); + } }); - it("does not deliver an outer catalog event after nested invalidation", () => { - let invalidate: - | ((event: SqlCatalogInvalidation) => void) - | undefined; + it("retains auxiliary owners across unrelated context changes", () => { const service = createSqlLanguageService({ - catalog: catalogProvider( - async () => ({ - coverage: { kind: "complete" }, - epoch: { generation: 0, token: "initial" }, + columns: { + id: "columns", + loadColumns: async () => ({ + epoch: { generation: 1, token: "epoch-1" }, relations: [], - status: "ready", }), - (_scope, listener) => { - invalidate = listener; - return () => undefined; - }, - ), + }, dialects: [duckdb], + namespaces: { + id: "namespaces", + search: async () => ({ + containers: [], + coverage: "complete", + epoch: { generation: 1, token: "epoch-1" }, + status: "ready", + }), + }, }); const session = service.openDocument({ context: { - catalog: { scope: "connection:1" }, + catalog: { scope: "connection:stable" }, dialect: "duckdb", - engine: "local", + engine: "before", }, - text: "SELECT * FROM ", - }); - const events: SqlRevision[] = []; - let nested = false; - session.onDidChange((event) => { - events.push(event.revision); - if (!nested) { - nested = true; - invalidate?.({ - epoch: { generation: 2, token: "nested" }, - }); - } - }); - session.onDidChange((event) => { - events.push(event.revision); + text: "SELECT 1", }); - - invalidate?.({ - epoch: { generation: 1, token: "outer" }, + const revision = session.update({ + baseRevision: session.revision, + context: { + catalog: { scope: "connection:stable" }, + dialect: "duckdb", + engine: "after", + }, }); - expect(events).toHaveLength(4); - expect(events[0]).toBe(events[1]); - expect(events[1]).not.toBe(events[2]); - expect(events[2]).toBe(events[3]); - expect(session.revision).toBe(events[2]); + expect(session.revision).toBe(revision); service.dispose(); }); - it("keeps duplicate listener subscriptions independent", () => { + it("invalidates auxiliary caches with the relation catalog epoch", async () => { + let generation = 1; + let columnCalls = 0; + let namespaceCalls = 0; let invalidate: | ((event: SqlCatalogInvalidation) => void) | undefined; const service = createSqlLanguageService({ - catalog: catalogProvider( - async () => ({ + catalog: { + id: "relations", + search: async () => ({ coverage: { kind: "complete" }, - epoch: { generation: 0, token: "initial" }, + epoch: { generation, token: `epoch-${generation}` }, relations: [], status: "ready", }), - (_scope, listener) => { + subscribe: (_scope, listener) => { invalidate = listener; return () => undefined; }, - ), + }, + columns: { + id: "columns", + loadColumns: async (request) => { + columnCalls += 1; + return { + epoch: { generation, token: `epoch-${generation}` }, + relations: request.relations.map((relation) => ({ + columns: [], + coverage: "complete", + relationEntityId: relation.requestKey, + requestKey: relation.requestKey, + status: "ready", + })), + }; + }, + }, dialects: [duckdb], - }); - const session = service.openDocument({ - context: { - catalog: { scope: "connection:1" }, - dialect: "duckdb", - engine: "local", + namespaces: { + id: "namespaces", + search: async () => { + namespaceCalls += 1; + return { + containers: [], + coverage: "complete", + epoch: { generation, token: `epoch-${generation}` }, + status: "ready", + }; + }, }, - text: "SELECT * FROM ", }); - const events: string[] = []; - const listener = (event: SqlSessionChangeEvent): void => { - events.push(event.reason); - }; - const first = session.onDidChange(listener); - const second = session.onDidChange(listener); + const open = (text: string) => + service.openDocument({ + context: { + catalog: { scope: "connection:invalidate" }, + dialect: "duckdb", + engine: "local", + }, + text, + }); + const columns = open("SELECT u. FROM users u"); + const namespaces = open("SELECT * FROM ma"); + const completeColumns = () => + columns.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + const completeNamespaces = () => + namespaces.complete({ + position: "SELECT * FROM ma".length, + trigger: { kind: "invoked" }, + }); - first.dispose(); - invalidate?.({ - epoch: { generation: 1, token: "first" }, + await completeColumns(); + await completeNamespaces(); + await completeColumns(); + await completeNamespaces(); + expect({ columnCalls, namespaceCalls }).toEqual({ + columnCalls: 1, + namespaceCalls: 1, }); - expect(events).toEqual(["catalog"]); - second.dispose(); + generation = 2; invalidate?.({ - epoch: { generation: 2, token: "second" }, - }); - expect(events).toEqual(["catalog"]); - service.dispose(); - }); - - it("validates completion configuration and request input", async () => { - expectSessionError("invalid-service-options", () => { - createSqlLanguageService({ - completion: { catalogResponseBudgetMs: 51 }, - dialects: [duckdb], - }); + epoch: { generation, token: `epoch-${generation}` }, }); - const { service, session } = openSession(); - await expect( - session.complete(null as never), - ).rejects.toMatchObject({ - code: "invalid-completion-request", + await completeColumns(); + await completeNamespaces(); + expect({ columnCalls, namespaceCalls }).toEqual({ + columnCalls: 2, + namespaceCalls: 2, }); service.dispose(); }); - it("rejects catalog context without a configured provider atomically", () => { + it("manually invalidates auxiliary-only catalog authority", async () => { + let generation = 1; + let columnCalls = 0; + let namespaceCalls = 0; const service = createSqlLanguageService({ + columns: { + id: "columns", + loadColumns: async (request) => { + columnCalls += 1; + return { + epoch: { generation, token: `epoch-${generation}` }, + relations: request.relations.map((relation) => ({ + columns: [], + coverage: "complete", + relationEntityId: relation.requestKey, + requestKey: relation.requestKey, + status: "ready", + })), + }; + }, + }, dialects: [duckdb], - }); - expectSessionError("invalid-context", () => { - service.openDocument({ - context: { - catalog: { scope: "connection:1" }, - dialect: "duckdb", - engine: "local", + namespaces: { + id: "namespaces", + search: async () => { + namespaceCalls += 1; + return { + containers: [], + coverage: "complete", + epoch: { generation, token: `epoch-${generation}` }, + status: "ready", + }; }, - text: "SELECT * FROM ", - }); + }, }); - const session = service.openDocument({ + const columns = service.openDocument({ context: { + catalog: { scope: "connection:manual-invalidate" }, dialect: "duckdb", engine: "local", }, - text: "SELECT * FROM ", + text: "SELECT u. FROM users u", }); - const revision = session.revision; - expectSessionError("invalid-context", () => { - session.update({ - baseRevision: revision, - context: { - catalog: { scope: "connection:1" }, - dialect: "duckdb", - engine: "local", - }, - }); + const namespaces = service.openDocument({ + context: { + catalog: { scope: "connection:manual-invalidate" }, + dialect: "duckdb", + engine: "local", + }, + text: "SELECT * FROM ma", }); - expect(session.revision).toBe(revision); + const completeColumns = () => + columns.complete({ + position: "SELECT u.".length, + trigger: { kind: "invoked" }, + }); + const completeNamespaces = () => + namespaces.complete({ + position: "SELECT * FROM ma".length, + trigger: { kind: "invoked" }, + }); + + await completeColumns(); + await completeNamespaces(); + await completeColumns(); + await completeNamespaces(); + expect({ columnCalls, namespaceCalls }).toEqual({ + columnCalls: 1, + namespaceCalls: 1, + }); + + generation = 2; + const columnRevision = columns.invalidateCatalog(); + const namespaceRevision = namespaces.invalidateCatalog(); + expect(columns.revision).toBe(columnRevision); + expect(namespaces.revision).toBe(namespaceRevision); + await completeColumns(); + await completeNamespaces(); + expect({ columnCalls, namespaceCalls }).toEqual({ + columnCalls: 2, + namespaceCalls: 2, + }); + + columns.dispose(); + expect(() => columns.invalidateCatalog()).toThrow( + "SQL document session is disposed", + ); + namespaces.dispose(); service.dispose(); }); }); @@ -3550,6 +4625,18 @@ describe("session coverage hardening", () => { }); }); + it.each([ + { columns: { id: "", loadColumns: async () => ({}) } }, + { namespaces: { id: "", search: async () => ({}) } }, + ])("rejects a malformed auxiliary provider %#", (provider) => { + expectSessionError("invalid-service-options", () => { + Reflect.apply(createSqlLanguageService, undefined, [{ + ...provider, + dialects: [duckdb], + }]); + }); + }); + it.each([ { expectedIssue: "catalog-failed", @@ -4014,6 +5101,15 @@ describe("session coverage hardening", () => { }, completion: { catalogResponseBudgetMs: 0 }, dialects: [duckdb], + namespaces: { + id: "retained-intent-namespaces", + search: async () => ({ + containers: [], + coverage: "complete", + epoch: { generation: 0, token: "ready" }, + status: "ready", + }), + }, }); const session = service.openDocument({ context: { diff --git a/src/vnext/browser_tests/fixtures/node-sql-parser-crash-worker.js b/src/vnext/browser_tests/fixtures/node-sql-parser-crash-worker.js index 530f6a4..1ec2a4f 100644 --- a/src/vnext/browser_tests/fixtures/node-sql-parser-crash-worker.js +++ b/src/vnext/browser_tests/fixtures/node-sql-parser-crash-worker.js @@ -1,6 +1,6 @@ globalThis.postMessage({ kind: "ready", - protocolVersion: 1, + protocolVersion: 2, }); globalThis.addEventListener("message", () => { throw new Error("Intentional parser executor crash fixture"); diff --git a/src/vnext/browser_tests/fixtures/node-sql-parser-silent-worker.js b/src/vnext/browser_tests/fixtures/node-sql-parser-silent-worker.js index b2c2edd..7ae4382 100644 --- a/src/vnext/browser_tests/fixtures/node-sql-parser-silent-worker.js +++ b/src/vnext/browser_tests/fixtures/node-sql-parser-silent-worker.js @@ -1,5 +1,5 @@ globalThis.postMessage({ kind: "ready", - protocolVersion: 1, + protocolVersion: 2, }); globalThis.addEventListener("message", () => {}); diff --git a/src/vnext/browser_tests/node-sql-parser-browser-executor.test.ts b/src/vnext/browser_tests/node-sql-parser-browser-executor.test.ts index 08a6b9b..eac837f 100644 --- a/src/vnext/browser_tests/node-sql-parser-browser-executor.test.ts +++ b/src/vnext/browser_tests/node-sql-parser-browser-executor.test.ts @@ -177,6 +177,7 @@ test( ), ).resolves.toStrictEqual({ kind: "parsed", + queryBindings: expect.any(Object), statementKind: "query", }); await expect( @@ -187,6 +188,7 @@ test( ), ).resolves.toStrictEqual({ kind: "parsed", + queryBindings: expect.any(Object), statementKind: "query", }); await expect( @@ -197,6 +199,7 @@ test( ), ).resolves.toStrictEqual({ kind: "parsed", + queryBindings: expect.any(Object), statementKind: "query", }); } finally { @@ -232,6 +235,7 @@ test( submitQuery(executor, "postgresql", "SELECT 2"), ).resolves.toStrictEqual({ kind: "parsed", + queryBindings: expect.any(Object), statementKind: "query", }); expect(generation).toBe(2); @@ -268,6 +272,7 @@ test( submitQuery(executor, "bigquery", "SELECT 2"), ).resolves.toStrictEqual({ kind: "parsed", + queryBindings: expect.any(Object), statementKind: "query", }); expect(generation).toBe(2); diff --git a/src/vnext/browser_tests/node-sql-parser-browser-worker.test.ts b/src/vnext/browser_tests/node-sql-parser-browser-worker.test.ts index 714a853..968dcf6 100644 --- a/src/vnext/browser_tests/node-sql-parser-browser-worker.test.ts +++ b/src/vnext/browser_tests/node-sql-parser-browser-worker.test.ts @@ -151,6 +151,7 @@ test( expect(firstPostgresql).toStrictEqual({ kind: "parsed", protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, + queryBindings: expect.any(Object), requestId: 1, statementKind: "query", }); @@ -166,6 +167,7 @@ test( ).resolves.toStrictEqual({ kind: "parsed", protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, + queryBindings: expect.any(Object), requestId: 2, statementKind: "query", }); @@ -180,6 +182,7 @@ test( ).resolves.toStrictEqual({ kind: "parsed", protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, + queryBindings: expect.any(Object), requestId: 3, statementKind: "query", }); diff --git a/src/vnext/codemirror/__tests__/sql-editor.test.ts b/src/vnext/codemirror/__tests__/sql-editor.test.ts index 39ebef4..063c6aa 100644 --- a/src/vnext/codemirror/__tests__/sql-editor.test.ts +++ b/src/vnext/codemirror/__tests__/sql-editor.test.ts @@ -11,6 +11,7 @@ import { EditorSelection, type Extension } from "@codemirror/state"; import { EditorView } from "@codemirror/view"; import { afterEach, describe, expect, it, vi } from "vitest"; import { + bigQueryDialect, createSqlLanguageService, duckdbDialect, type SqlCatalogSearchRequest, @@ -27,6 +28,7 @@ import { type SqlRelationCatalogProvider, type SqlRevision, type SqlSessionChangeEvent, + type SqlTextRange, } from "../../index.js"; import { createSqlCompletionRefreshToken, @@ -46,8 +48,11 @@ interface FakeServiceHarness { readonly completeSignals: AbortSignal[]; readonly emit: (event: SqlSessionChangeEvent) => void; readonly getLastToken: () => SqlCompletionRefreshToken | null; + readonly invalidations: () => number; readonly service: SqlLanguageService; readonly sessionDisposals: () => number; + readonly statementBoundaryCalls: () => number; + readonly statementIntersectionCalls: () => number; readonly updates: readonly SqlDocumentUpdate[]; } @@ -126,6 +131,7 @@ function fakeService( token: SqlCompletionRefreshToken, ) => SqlCompletionResult | Promise, rejectUpdates = false, + statementCode: SqlTextRange | null = null, ): FakeServiceHarness { const completeSignals: AbortSignal[] = []; const updates: SqlDocumentUpdate[] = []; @@ -133,7 +139,10 @@ function fakeService( | ((event: SqlSessionChangeEvent) => void) | null = null; let lastToken: SqlCompletionRefreshToken | null = null; + let invalidationCount = 0; let sessionDisposalCount = 0; + let statementBoundaryCallCount = 0; + let statementIntersectionCallCount = 0; const service: SqlLanguageService = { dispose: () => undefined, openDocument: () => { @@ -154,6 +163,11 @@ function fakeService( disposed = true; sessionDisposalCount += 1; }, + invalidateCatalog: () => { + invalidationCount += 1; + revision = createSqlRevisionToken(); + return revision; + }, isCurrent: (candidate) => !disposed && candidate === revision, onDidChange: (nextListener) => { listener = nextListener; @@ -166,30 +180,56 @@ function fakeService( get revision() { return revision; }, - statementBoundaryAt: () => ({ - boundary: { - boundaryQuality: "exact", - code: null, - endState: { kind: "normal" }, - extent: { from: 0, to: 0 }, - hasCode: false, - source: { from: 0, to: 0 }, - terminator: null, - }, - revision, - }), - statementBoundariesIntersecting: () => ({ - boundaries: [{ - boundaryQuality: "exact", - code: null, - endState: { kind: "normal" }, - extent: { from: 0, to: 0 }, - hasCode: false, - source: { from: 0, to: 0 }, - terminator: null, - }], - revision, - }), + statementBoundaryAt: () => { + statementBoundaryCallCount += 1; + return { + boundary: statementCode === null + ? { + boundaryQuality: "exact", + code: null, + endState: { kind: "normal" }, + extent: { from: 0, to: 0 }, + hasCode: false, + source: { from: 0, to: 0 }, + terminator: null, + } + : { + boundaryQuality: "exact", + code: statementCode, + endState: { kind: "normal" }, + extent: statementCode, + hasCode: true, + source: statementCode, + terminator: null, + }, + revision, + }; + }, + statementBoundariesIntersecting: () => { + statementIntersectionCallCount += 1; + return { + boundaries: [statementCode === null + ? { + boundaryQuality: "exact", + code: null, + endState: { kind: "normal" }, + extent: { from: 0, to: 0 }, + hasCode: false, + source: { from: 0, to: 0 }, + terminator: null, + } + : { + boundaryQuality: "exact", + code: statementCode, + endState: { kind: "normal" }, + extent: statementCode, + hasCode: true, + source: statementCode, + terminator: null, + }], + revision, + }; + }, update: (update) => { updates.push(update); if (rejectUpdates) { @@ -206,8 +246,12 @@ function fakeService( completeSignals, emit: (event) => listener?.(event), getLastToken: () => lastToken, + invalidations: () => invalidationCount, service, sessionDisposals: () => sessionDisposalCount, + statementBoundaryCalls: () => statementBoundaryCallCount, + statementIntersectionCalls: () => + statementIntersectionCallCount, updates, }; } @@ -320,6 +364,339 @@ async function resolveCompletionInfo( } describe("sqlEditor", () => { + it("exposes session controls only for owned views", () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + }); + const view = createView(support.extension, "SELECT 1;SELECT 2"); + const foreign = createView([]); + + expect(support.statementBoundaryAt(view, { + affinity: "left", + position: 9, + })?.boundary).toMatchObject({ + boundaryQuality: "exact", + code: { from: 0, to: 8 }, + hasCode: true, + }); + expect(support.statementBoundariesIntersecting(view, { + from: 0, + to: view.state.doc.length, + })?.boundaries).toHaveLength(2); + expect(support.invalidateCatalog(view)).not.toBeNull(); + expect(support.invalidateCatalog(foreign)).toBeNull(); + expect(support.statementBoundaryAt(foreign, { + affinity: "left", + position: 0, + })).toBeNull(); + expect(support.statementBoundariesIntersecting(foreign, { + from: 0, + to: 0, + })).toBeNull(); + expect(support.statementBoundaryAt(view, { + affinity: "left", + position: 100, + })).toBeNull(); + + view.destroy(); + expect(support.invalidateCatalog(view)).toBeNull(); + expect(support.statementBoundaryAt(view, { + affinity: "left", + position: 0, + })).toBeNull(); + expect(support.statementBoundariesIntersecting(view, { + from: 0, + to: 0, + })).toBeNull(); + service.dispose(); + }); + + it("proxies catalog invalidation only to its owned live session", () => { + const harness = fakeService((revision) => readyResult(revision, [])); + const support = sqlEditor({ + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + const foreign = createView([]); + + expect(support.invalidateCatalog(view)).not.toBeNull(); + expect(harness.invalidations()).toBe(1); + expect(support.invalidateCatalog(foreign)).toBeNull(); + expect(harness.invalidations()).toBe(1); + view.destroy(); + expect(support.invalidateCatalog(view)).toBeNull(); + expect(harness.invalidations()).toBe(1); + }); + + it("renders an opt-in visible-line statement gutter", async () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: {}, + }); + const view = createView( + support.extension, + "SELECT 1;\n\n/* separator */\nSELECT 2;", + ); + + await vi.waitFor(() => { + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(2); + }); + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker-active"), + ).toHaveLength(1); + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker-inactive"), + ).toHaveLength(1); + expect(Array.from( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).findIndex((marker) => + marker.classList.contains("cm-sql-statement-marker-active") + )).toBe(1); + + view.dispatch({ selection: { anchor: 1 } }); + await vi.waitFor(() => { + const markers = view.dom.querySelectorAll( + ".cm-sql-statement-marker", + ); + expect(markers).toHaveLength(2); + expect( + view.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ), + ).toHaveLength(1); + expect(Array.from(markers).findIndex((marker) => + marker.classList.contains( + "cm-sql-statement-marker-active", + ) + )).toBe(0); + }); + service.dispose(); + }); + + it("does not install a statement gutter by default", () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + }); + const view = createView(support.extension, "SELECT 1"); + + expect( + view.dom.querySelector(".cm-sql-statement-gutter"), + ).toBeNull(); + service.dispose(); + }); + + it("renders no marker for an empty document", () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: {}, + }); + const view = createView(support.extension, ""); + + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(0); + service.dispose(); + }); + + it("supports hidden and active-only gutter policies", async () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const hidden = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: { hideWhenNotFocused: true }, + }); + const hiddenView = createView(hidden.extension, "SELECT 1"); + expect( + hiddenView.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(0); + hiddenView.focus(); + await vi.waitFor(() => { + expect( + hiddenView.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(1); + }); + + const activeOnly = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: { showInactive: false }, + }); + const activeView = createView( + activeOnly.extension, + "SELECT 1;\nSELECT 2", + ); + await vi.waitFor(() => { + expect( + activeView.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ), + ).toHaveLength(1); + expect( + activeView.dom.querySelectorAll( + ".cm-sql-statement-marker-inactive", + ), + ).toHaveLength(0); + }); + activeOnly.setContext(activeView, { + dialect: "duckdb", + engine: "remote", + }); + await vi.waitFor(() => { + expect( + activeView.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ), + ).toHaveLength(1); + }); + service.dispose(); + }); + + it("shows only inactive markers when no code boundary is current", async () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: {}, + }); + const view = createView( + support.extension, + "SELECT 1;\n/* trailing */", + ); + + await vi.waitFor(() => { + expect( + view.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ), + ).toHaveLength(0); + expect( + view.dom.querySelectorAll( + ".cm-sql-statement-marker-inactive", + ), + ).toHaveLength(1); + }); + service.dispose(); + expect(() => { + view.dispatch({ selection: { anchor: 0 } }); + }).not.toThrow(); + }); + + it("accepts an explicit false gutter option", () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: false, + }); + const view = createView(support.extension, "SELECT 1"); + expect( + view.dom.querySelector(".cm-sql-statement-gutter"), + ).toBeNull(); + service.dispose(); + }); + + it("marks internal blank lines but not separator trivia", async () => { + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service, + statementGutter: {}, + }); + const view = createView( + support.extension, + "SELECT\n\n 1;\n\nSELECT 2", + ); + + await vi.waitFor(() => { + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(4); + }); + service.dispose(); + }); + + it("does not fall back across an opaque right boundary", async () => { + const service = createSqlLanguageService({ + dialects: [bigQueryDialect()], + }); + const support = sqlEditor({ + initialContext: { dialect: "bigquery", engine: "local" }, + service, + statementGutter: {}, + }); + const documentText = + "SELECT 1; IF condition THEN SELECT 2; END IF;"; + const sharedBoundary = documentText.indexOf(";") + 1; + const view = createView(support.extension, documentText); + view.dispatch({ selection: { anchor: sharedBoundary } }); + + expect( + support.statementBoundaryAt(view, { + affinity: "right", + position: sharedBoundary, + })?.boundary.boundaryQuality, + ).toBe("opaque"); + await vi.waitFor(() => { + expect( + view.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ), + ).toHaveLength(0); + }); + service.dispose(); + }); + + it("queries structural boundaries once per relevant redraw", () => { + const documentText = "SELECT\n\n 1;\nSELECT 2"; + const harness = fakeService( + (revision) => readyResult(revision, []), + false, + { from: 0, to: documentText.length }, + ); + const support = sqlEditor({ + initialContext: { dialect: "duckdb", engine: "local" }, + service: harness.service, + statementGutter: {}, + }); + const view = createView(support.extension, documentText); + + expect(harness.statementBoundaryCalls()).toBe(1); + expect(harness.statementIntersectionCalls()).toBe(1); + view.dispatch({ selection: { anchor: 1 } }); + expect(harness.statementBoundaryCalls()).toBe(2); + expect(harness.statementIntersectionCalls()).toBe(2); + view.dispatch({}); + expect(harness.statementBoundaryCalls()).toBe(2); + expect(harness.statementIntersectionCalls()).toBe(2); + }); + it("maps current service completions and applies the exact core edit", async () => { const service = createSqlLanguageService({ catalog: { @@ -748,6 +1125,32 @@ describe("sqlEditor", () => { expect(harness.sessionDisposals()).toBe(0); }); + it("refuses a completion edit that overlaps an embedded region", async () => { + const harness = fakeService((revision) => + readyResult(revision, [completionItem(14, 16)]) + ); + const support = sqlEditor({ + initialContext: context(), + initialEmbeddedRegions: [{ + from: 15, + language: "host", + to: 16, + }], + service: harness.service, + }); + const view = createView(support.extension); + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + const completion = currentCompletions(view.state)[0]; + if (!completion || typeof completion.apply !== "function") { + throw new Error("Expected completion apply callback"); + } + + completion.apply(view, completion, 14, 16); + expect(view.state.doc.toString()).toBe("SELECT * FROM us"); + expect(harness.updates).toEqual([]); + }); + it("combines document and final context effects into one current input", async () => { const requests: SqlCatalogSearchRequest[] = []; const service = createSqlLanguageService({ @@ -927,6 +1330,294 @@ describe("sqlEditor", () => { ); }); + it("maps column and namespace completion presentation", async () => { + const epoch = { generation: 1, token: "epoch-1" }; + const items: readonly SqlCompletionItem[] = [{ + edit: { from: 14, insert: "users_column", to: 16 }, + kind: "column", + label: "users_column", + provenance: { + columnEntityId: "users:name", + epoch, + kind: "column-catalog", + providerId: "columns", + relationEntityId: "users", + scope: "connection:fake", + }, + relationRequestKey: "binding:0", + }, { + edit: { from: 14, insert: "users_namespace", to: 16 }, + kind: "namespace", + label: "users_namespace", + provenance: { + containerEntityId: "schema:main", + epoch, + kind: "namespace-catalog", + providerId: "namespaces", + scope: "connection:fake", + }, + role: "schema", + }]; + const harness = fakeService((revision) => + readyResult(revision, items) + ); + const support = sqlEditor({ + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + expect(currentCompletions(view.state)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + label: "users_column", + type: "property", + }), + expect.objectContaining({ + label: "users_namespace", + type: "namespace", + }), + ]), + ); + }); + + it("denies SQL completion at an unmatched template EOF without removing external sources", async () => { + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const gate = vi.fn((state, position) => + !state.sliceDoc(0, position).endsWith("{") + ); + const support = sqlEditor({ + autocomplete: { + externalSources: [() => ({ + from: 15, + options: [{ label: "python_variable" }], + })], + isCompletionPositionAllowed: gate, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension, "SELECT * FROM {"); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + expect(harness.completeSignals).toHaveLength(0); + expect(currentCompletions(view.state)).toEqual([ + expect.objectContaining({ label: "python_variable" }), + ]); + expect(gate).toHaveBeenCalledWith(view.state, 15); + }); + + it("allows normal SQL completion through the position gate", async () => { + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const gate = vi.fn(() => true); + const support = sqlEditor({ + autocomplete: { + isCompletionPositionAllowed: gate, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + expect(harness.completeSignals).toHaveLength(1); + expect(currentCompletions(view.state)).toEqual([ + expect.objectContaining({ label: "users" }), + ]); + expect(gate).toHaveBeenCalledWith(view.state, 16); + }); + + it("fails a throwing position gate closed while retaining external sources", async () => { + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const support = sqlEditor({ + autocomplete: { + externalSources: [() => ({ + from: 15, + options: [{ label: "python_variable" }], + })], + isCompletionPositionAllowed: () => { + throw new Error("host state unavailable"); + }, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension, "SELECT * FROM {"); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + expect(harness.completeSignals).toHaveLength(0); + expect(currentCompletions(view.state)).toEqual([ + expect.objectContaining({ label: "python_variable" }), + ]); + }); + + it("cancels pending SQL completion when the position gate flips false", async () => { + let allowed = true; + const harness = fakeService( + () => new Promise(() => undefined), + ); + const support = sqlEditor({ + autocomplete: { + isCompletionPositionAllowed: () => allowed, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await vi.waitFor(() => { + expect(harness.completeSignals).toHaveLength(1); + }); + allowed = false; + view.dispatch({}); + await vi.waitFor(() => { + expect(harness.completeSignals[0]?.aborted).toBe(true); + expect(currentCompletions(view.state)).toEqual([]); + }); + }); + + it("restarts external sources after the SQL gate closes stale options", async () => { + let allowed = true; + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const support = sqlEditor({ + autocomplete: { + externalSources: [() => ({ + from: 16, + options: [{ label: "external_variable" }], + })], + isCompletionPositionAllowed: () => allowed, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + expect(currentCompletions(view.state).map((item) => item.label).sort()) + .toEqual(["external_variable", "users"]); + + allowed = false; + view.dispatch({}); + await vi.waitFor(() => { + expect(currentCompletions(view.state).map((item) => item.label)) + .toEqual(["external_variable"]); + }); + }); + + it("closes SQL options denied while the provider is resolving", async () => { + const runtime = controlledRuntime(); + let allowed = true; + let resolveResult: + | ((result: SqlCompletionResult) => void) + | undefined; + let revision: SqlRevision | null = null; + const harness = fakeService((currentRevision) => { + revision = currentRevision; + return new Promise((resolve) => { + resolveResult = resolve; + }); + }); + const support = createSqlEditorInternal({ + autocomplete: { + closeOnBlur: false, + isCompletionPositionAllowed: () => allowed, + }, + initialContext: context(), + service: harness.service, + }, runtime.runtime); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await vi.waitFor(() => + expect(harness.completeSignals).toHaveLength(1) + ); + if (revision === null) { + throw new Error("Expected deferred completion revision"); + } + allowed = false; + resolveResult?.(readyResult(revision, [completionItem()])); + await vi.waitFor(() => expect(runtime.queued).toHaveLength(1)); + + runtime.queued[0]?.(); + expect(runtime.closes).toEqual([view]); + }); + + it("does not let a stale gate-close task affect a newer editor state", async () => { + const runtime = controlledRuntime(); + let allowed = true; + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const support = createSqlEditorInternal({ + autocomplete: { + closeOnBlur: false, + isCompletionPositionAllowed: () => allowed, + }, + initialContext: context(), + service: harness.service, + }, runtime.runtime); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + allowed = false; + view.dispatch({}); + expect(runtime.queued).toHaveLength(1); + + view.dispatch({ selection: { anchor: 15 } }); + runtime.queued[0]?.(); + expect(runtime.closes).toEqual([]); + }); + + it("disposes rich info when the position gate flips false", async () => { + let allowed = true; + const destroys: Array> = []; + const harness = fakeService((revision) => + readyResult(revision, [completionItem()]) + ); + const support = sqlEditor({ + autocomplete: { + infoResolver: () => { + const destroy = vi.fn(); + destroys.push(destroy); + return { + destroy, + dom: document.createElement("div"), + }; + }, + isCompletionPositionAllowed: () => allowed, + }, + initialContext: context(), + service: harness.service, + }); + const view = createView(support.extension); + + expect(startCompletion(view)).toBe(true); + await waitForActiveCompletion(view); + await vi.waitFor(() => expect(destroys.length).toBeGreaterThan(0)); + const currentDestroy = destroys.at(-1); + if (!currentDestroy) throw new Error("Expected rich info"); + + allowed = false; + view.dispatch({}); + expect(currentDestroy).toHaveBeenCalledTimes(1); + }); + it.each([ "cancelled", "failed", diff --git a/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts new file mode 100644 index 0000000..3c9fd35 --- /dev/null +++ b/src/vnext/codemirror/browser_tests/sql-editor-completion-gate.test.ts @@ -0,0 +1,345 @@ +import { + currentCompletions, + startCompletion, +} from "@codemirror/autocomplete"; +import { EditorView } from "@codemirror/view"; +import { expect, onTestFinished, test } from "vitest"; +import { + createSqlLanguageService, + duckdbDialect, +} from "../../index.js"; +import { sqlEditor } from "../index.js"; + +test("vNext completion gate routes unmatched template EOF to external sources", async () => { + const parent = document.createElement("div"); + document.body.append(parent); + onTestFinished(() => parent.remove()); + let catalogSearches = 0; + const service = createSqlLanguageService({ + catalog: { + id: "browser-catalog", + search: async () => { + catalogSearches += 1; + return { + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "ready" }, + relations: [{ + canonicalPath: [ + { + quoted: false, + role: "schema", + value: "main", + }, + { + quoted: false, + role: "relation", + value: "users", + }, + ], + completionPathStart: 1, + entityId: "main.users", + matchQuality: "exact", + relationKind: "table", + }], + status: "ready" as const, + }; + }, + }, + dialects: [duckdbDialect()], + }); + onTestFinished(() => service.dispose()); + const support = sqlEditor({ + autocomplete: { + externalSources: [(context) => ({ + from: context.pos, + options: [{ label: "python_variable" }], + })], + isCompletionPositionAllowed: (state, position) => { + const prefix = state.sliceDoc(0, position); + return prefix.lastIndexOf("{") <= prefix.lastIndexOf("}"); + }, + }, + initialContext: { + catalog: { + scope: "browser", + searchPath: [[{ quoted: false, value: "main" }]], + }, + dialect: "duckdb", + }, + initialEmbeddedRegions: [{ + from: 14, + language: "python", + to: 15, + }], + service, + }); + const view = new EditorView({ + doc: "SELECT * FROM {", + extensions: support.extension, + parent, + selection: { anchor: 15 }, + }); + onTestFinished(() => view.destroy()); + + expect(startCompletion(view)).toBe(true); + await expect.poll(() => + currentCompletions(view.state).map((item) => item.label) + ).toEqual(["python_variable"]); + expect(catalogSearches).toBe(0); +}); + +test("vNext completion gate permits normal SQL in a browser editor", async () => { + const parent = document.createElement("div"); + document.body.append(parent); + onTestFinished(() => parent.remove()); + const service = createSqlLanguageService({ + catalog: { + id: "browser-catalog", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "ready" }, + relations: [{ + canonicalPath: [ + { + quoted: false, + role: "schema", + value: "main", + }, + { + quoted: false, + role: "relation", + value: "users", + }, + ], + completionPathStart: 1, + entityId: "main.users", + matchQuality: "exact", + relationKind: "table", + }], + status: "ready" as const, + }), + }, + dialects: [duckdbDialect()], + }); + onTestFinished(() => service.dispose()); + const support = sqlEditor({ + autocomplete: { + isCompletionPositionAllowed: () => true, + }, + initialContext: { + catalog: { + scope: "browser", + searchPath: [[{ quoted: false, value: "main" }]], + }, + dialect: "duckdb", + }, + service, + }); + const documentText = "SELECT * FROM us"; + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + selection: { anchor: documentText.length }, + }); + onTestFinished(() => view.destroy()); + + expect(startCompletion(view)).toBe(true); + await expect.poll(() => + currentCompletions(view.state).map((item) => item.label) + ).toContain("users"); +}); + +test("vNext completion gate preserves external sources when it closes SQL options", async () => { + const parent = document.createElement("div"); + document.body.append(parent); + onTestFinished(() => parent.remove()); + let allowed = true; + const service = createSqlLanguageService({ + catalog: { + id: "browser-gate-transition", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 0, token: "ready" }, + relations: [{ + canonicalPath: [ + { + quoted: false, + role: "schema", + value: "main", + }, + { + quoted: false, + role: "relation", + value: "users", + }, + ], + completionPathStart: 1, + entityId: "main.users", + matchQuality: "exact", + relationKind: "table", + }], + status: "ready" as const, + }), + }, + dialects: [duckdbDialect()], + }); + onTestFinished(() => service.dispose()); + const support = sqlEditor({ + autocomplete: { + externalSources: [(context) => ({ + from: context.pos, + options: [{ label: "python_variable" }], + })], + isCompletionPositionAllowed: () => allowed, + }, + initialContext: { + catalog: { scope: "browser-gate-transition" }, + dialect: "duckdb", + }, + service, + }); + const documentText = "SELECT * FROM us"; + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + selection: { anchor: documentText.length }, + }); + onTestFinished(() => view.destroy()); + + expect(startCompletion(view)).toBe(true); + await expect.poll(() => + currentCompletions(view.state).map((item) => item.label).sort() + ).toEqual(["python_variable", "users"]); + + allowed = false; + view.dispatch({}); + await expect.poll(() => + currentCompletions(view.state).map((item) => item.label) + ).toEqual(["python_variable"]); +}); + +test("vNext editor applies a batched column completion", async () => { + const parent = document.createElement("div"); + document.body.append(parent); + onTestFinished(() => parent.remove()); + let columnCalls = 0; + const service = createSqlLanguageService({ + catalog: { + id: "browser-relations", + search: async () => ({ + coverage: { kind: "complete" }, + epoch: { generation: 1, token: "epoch-1" }, + relations: [], + status: "ready" as const, + }), + }, + columns: { + id: "browser-columns", + loadColumns: async (request) => { + columnCalls += 1; + const relation = request.relations[0]; + if (!relation) throw new Error("Expected one relation"); + return { + epoch: { generation: 1, token: "epoch-1" }, + relations: [{ + columns: [{ + columnEntityId: "users:name", + dataType: "VARCHAR", + identifier: { quoted: false, value: "name" }, + insertText: "name", + ordinal: 0, + }], + coverage: "complete", + relationEntityId: "users", + requestKey: relation.requestKey, + status: "ready", + }], + }; + }, + }, + dialects: [duckdbDialect()], + }); + onTestFinished(() => service.dispose()); + const support = sqlEditor({ + initialContext: { + catalog: { scope: "browser-columns" }, + dialect: "duckdb", + }, + service, + }); + const documentText = "SELECT u.na FROM users u"; + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + selection: { anchor: "SELECT u.na".length }, + }); + onTestFinished(() => view.destroy()); + + expect(startCompletion(view)).toBe(true); + await expect.poll(() => + currentCompletions(view.state).map((item) => ({ + label: item.label, + type: item.type, + })) + ).toEqual([{ label: "name", type: "property" }]); + expect(columnCalls).toBe(1); +}); + +test("vNext editor exposes namespace containers at relation sites", async () => { + const parent = document.createElement("div"); + document.body.append(parent); + onTestFinished(() => parent.remove()); + let namespaceCalls = 0; + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + namespaces: { + id: "browser-namespaces", + search: async () => { + namespaceCalls += 1; + return { + containers: [{ + canonicalPath: [{ + quoted: false, + role: "schema", + value: "main", + }], + containerEntityId: "schema:main", + insertText: "main", + matchQuality: "exact", + }], + coverage: "complete", + epoch: { generation: 1, token: "epoch-1" }, + status: "ready", + }; + }, + }, + }); + onTestFinished(() => service.dispose()); + const support = sqlEditor({ + initialContext: { + catalog: { scope: "browser-namespaces" }, + dialect: "duckdb", + }, + service, + }); + const documentText = "SELECT * FROM ma"; + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + selection: { anchor: documentText.length }, + }); + onTestFinished(() => view.destroy()); + + expect(startCompletion(view)).toBe(true); + await expect.poll(() => + currentCompletions(view.state).map((item) => ({ + label: item.label, + type: item.type, + })) + ).toEqual([{ label: "main", type: "namespace" }]); + expect(namespaceCalls).toBe(1); +}); diff --git a/src/vnext/codemirror/browser_tests/statement-gutter.test.ts b/src/vnext/codemirror/browser_tests/statement-gutter.test.ts new file mode 100644 index 0000000..2ba440b --- /dev/null +++ b/src/vnext/codemirror/browser_tests/statement-gutter.test.ts @@ -0,0 +1,120 @@ +import { EditorView } from "@codemirror/view"; +import { expect, onTestFinished, test } from "vitest"; +import { + createSqlLanguageService, + duckdbDialect, +} from "../../index.js"; +import { sqlEditor } from "../index.js"; + +test("vNext statement gutter follows the current statement", async () => { + const parent = document.createElement("div"); + parent.style.height = "240px"; + document.body.append(parent); + onTestFinished(() => parent.remove()); + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + onTestFinished(() => service.dispose()); + const support = sqlEditor({ + initialContext: { dialect: "duckdb" }, + service, + statementGutter: {}, + }); + const documentText = "SELECT 1;\n\n/* separator */\nSELECT 2;"; + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + selection: { anchor: documentText.length }, + }); + onTestFinished(() => view.destroy()); + view.focus(); + + await expect.poll(() => + view.dom.querySelectorAll(".cm-sql-statement-marker").length + ).toBe(2); + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker-active"), + ).toHaveLength(1); + expect(Array.from( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).findIndex((marker) => + marker.classList.contains("cm-sql-statement-marker-active") + )).toBe(1); + + view.dispatch({ selection: { anchor: 1 } }); + await expect.poll(() => { + const markers = Array.from( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ); + return markers.findIndex((marker) => + marker.classList.contains("cm-sql-statement-marker-active") + ); + }).toBe(0); +}); + +test("vNext statement gutter virtualizes a tall focused editor", async () => { + const parent = document.createElement("div"); + parent.style.height = "120px"; + parent.style.setProperty( + "--cm-sql-statement-color", + "rgb(1, 2, 3)", + ); + document.body.append(parent); + onTestFinished(() => parent.remove()); + const service = createSqlLanguageService({ + dialects: [duckdbDialect()], + }); + onTestFinished(() => service.dispose()); + const support = sqlEditor({ + initialContext: { dialect: "duckdb" }, + service, + statementGutter: { hideWhenNotFocused: true }, + }); + const documentText = Array.from( + { length: 200 }, + (_, index) => `SELECT\n\n ${index};`, + ).join("\n/* separator */\n"); + const view = new EditorView({ + doc: documentText, + extensions: support.extension, + parent, + }); + onTestFinished(() => view.destroy()); + + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker"), + ).toHaveLength(0); + view.focus(); + await expect.poll(() => + view.dom.querySelectorAll(".cm-sql-statement-marker").length + ).toBeGreaterThan(0); + const initialCount = view.dom.querySelectorAll( + ".cm-sql-statement-marker", + ).length; + expect(initialCount).toBeLessThan(200); + const marker = view.dom.querySelector( + ".cm-sql-statement-marker", + ); + expect(marker).not.toBeNull(); + expect(getComputedStyle(marker!).width).toBe("3px"); + expect(getComputedStyle(marker!).backgroundColor).toBe( + "rgb(1, 2, 3)", + ); + + view.dispatch({ + effects: EditorView.scrollIntoView(documentText.length, { + y: "start", + }), + selection: { anchor: documentText.length }, + }); + await expect.poll(() => view.viewport.from).toBeGreaterThan(0); + await expect.poll(() => + view.dom.querySelectorAll( + ".cm-sql-statement-marker-active", + ).length + ).toBe(3); + expect( + view.dom.querySelectorAll(".cm-sql-statement-marker").length, + ).toBeLessThan(200); +}); diff --git a/src/vnext/codemirror/index.ts b/src/vnext/codemirror/index.ts index e5e1d6f..16e2265 100644 --- a/src/vnext/codemirror/index.ts +++ b/src/vnext/codemirror/index.ts @@ -4,6 +4,9 @@ export { type SqlEditorOptions, type SqlEditorSupport, } from "./sql-editor.js"; +export type { + SqlEditorStatementGutterOptions, +} from "./statement-gutter.js"; export type { SqlCompletionInfoResolver, SqlCompletionInfoResolverContext, diff --git a/src/vnext/codemirror/sql-editor.ts b/src/vnext/codemirror/sql-editor.ts index 652b143..9171149 100644 --- a/src/vnext/codemirror/sql-editor.ts +++ b/src/vnext/codemirror/sql-editor.ts @@ -15,6 +15,7 @@ import { Prec, StateEffect, StateField, + type EditorState, type EditorSelection, type Extension, type StateEffectType, @@ -36,6 +37,12 @@ import type { SqlCompletionResult, SqlCompletionTask, } from "../relation-completion-types.js"; +import type { + SqlStatementBoundariesIntersectingRequest, + SqlStatementBoundariesIntersectingResult, + SqlStatementBoundaryAtRequest, + SqlStatementBoundaryAtResult, +} from "../statement-boundary-types.js"; import type { SqlContextInput, SqlDocumentContext, @@ -45,6 +52,10 @@ import type { SqlRevision, SqlTextChange, } from "../types.js"; +import { + createSqlStatementGutter, + type SqlEditorStatementGutterOptions, +} from "./statement-gutter.js"; export interface SqlEditorAutocompleteOptions { readonly activateOnTyping?: boolean; @@ -53,6 +64,10 @@ export interface SqlEditorAutocompleteOptions { readonly defaultKeymap?: boolean; readonly externalSources?: readonly CompletionSource[]; readonly infoResolver?: SqlCompletionInfoResolver; + readonly isCompletionPositionAllowed?: ( + state: EditorState, + position: number, + ) => boolean; readonly maxRenderedOptions?: number; readonly selectOnOpen?: boolean; readonly updateSyncTime?: number; @@ -67,6 +82,9 @@ export interface SqlEditorOptions< | readonly SqlEmbeddedRegion[] | undefined; readonly service: SqlLanguageService; + readonly statementGutter?: + | false + | SqlEditorStatementGutterOptions; } export interface SqlEditorSupport< @@ -77,6 +95,15 @@ export interface SqlEditorSupport< readonly SqlEmbeddedRegion[] >; readonly extension: Extension; + readonly invalidateCatalog: (view: EditorView) => SqlRevision | null; + readonly statementBoundariesIntersecting: ( + view: EditorView, + request: SqlStatementBoundariesIntersectingRequest, + ) => SqlStatementBoundariesIntersectingResult | null; + readonly statementBoundaryAt: ( + view: EditorView, + request: SqlStatementBoundaryAtRequest, + ) => SqlStatementBoundaryAtResult | null; readonly setContext: ( view: EditorView, context: SqlContextInput, @@ -103,6 +130,7 @@ export interface SqlEditorRuntime { interface CompletionCapture { readonly contextGeneration: number; readonly document: Text; + readonly embeddedRegions: readonly SqlEmbeddedRegion[]; readonly selection: EditorSelection; } @@ -155,7 +183,12 @@ function loadingLeaseMs( result: Extract, ): number | null { for (const issue of result.value.issues) { - if (issue.reason === "catalog-loading") { + if ( + (issue.reason === "catalog-loading" || + issue.reason === "column-catalog-loading" || + issue.reason === "namespace-catalog-loading") && + typeof issue.remainingIntentLeaseMs === "number" + ) { return issue.remainingIntentLeaseMs; } } @@ -177,6 +210,8 @@ function haveOneEditRange(items: readonly SqlCompletionItem[]): boolean { } function completionType(item: SqlCompletionItem): string { + if (item.kind === "column") return "property"; + if (item.kind === "namespace") return "namespace"; return item.relationKind === "cte" ? "type" : "table"; } @@ -239,6 +274,7 @@ export function createSqlEditorInternal< const { externalSources = [], infoResolver, + isCompletionPositionAllowed, ...autocompleteOptions } = autocomplete; @@ -252,6 +288,7 @@ export function createSqlEditorInternal< #contextGeneration = 0; #destroyed = false; #disposingInfo = false; + #completionPositionAllowed: boolean; #hasEmbeddedRegions: boolean; #info: ActiveCompletionInfo | null = null; #intent: CompletionIntent | null = null; @@ -273,6 +310,11 @@ export function createSqlEditorInternal< text: view.state.doc.toString(), }); this.#hasEmbeddedRegions = initialRegions.length > 0; + this.#completionPositionAllowed = + this.#completionPositionIsAllowed( + view.state, + view.state.selection.main.head, + ); this.#lastCompletionStatus = completionStatus(view.state); this.#lastSelectedCompletion = selectedCompletion(view.state); this.#lastSelectedCompletionIndex = selectedCompletionIndex( @@ -354,10 +396,32 @@ export function createSqlEditorInternal< !this.#destroyed && capture.contextGeneration === this.#contextGeneration && capture.document === state.doc && + capture.embeddedRegions === state.field(embeddedRegionsField) && capture.selection.eq(state.selection) ); } + #capture(): CompletionCapture { + const state = this.#view.state; + return { + contextGeneration: this.#contextGeneration, + document: state.doc, + embeddedRegions: state.field(embeddedRegionsField), + selection: state.selection, + }; + } + + #completionPositionIsAllowed( + state: EditorState, + position: number, + ): boolean { + try { + return isCompletionPositionAllowed?.(state, position) ?? true; + } catch { + return false; + } + } + #scheduleClose(): void { const sequence = this.#sequence; runtime.queueMicrotask(() => { @@ -371,6 +435,31 @@ export function createSqlEditorInternal< }); } + #scheduleCompletionGateClose(capture: CompletionCapture): void { + runtime.queueMicrotask(() => { + if ( + !this.#captureIsCurrent(capture) || + this.#completionPositionIsAllowed( + this.#view.state, + this.#view.state.selection.main.head, + ) + ) { + return; + } + runtime.closeCompletion(this.#view); + if ( + externalSources.length > 0 && + this.#captureIsCurrent(capture) && + !this.#completionPositionIsAllowed( + this.#view.state, + this.#view.state.selection.main.head, + ) + ) { + runtime.startCompletion(this.#view); + } + }); + } + #scheduleRefresh(capture: CompletionCapture): void { if ( this.#refreshScheduled || @@ -518,11 +607,15 @@ export function createSqlEditorInternal< context: CompletionContext, ): Promise => { this.#clearCompletionState(); - const capture: CompletionCapture = { - contextGeneration: this.#contextGeneration, - document: this.#view.state.doc, - selection: this.#view.state.selection, - }; + if ( + !this.#completionPositionIsAllowed( + context.state, + context.pos, + ) + ) { + return null; + } + const capture = this.#capture(); const controller = new AbortController(); let task: SqlCompletionTask; try { @@ -565,6 +658,18 @@ export function createSqlEditorInternal< } return null; } + if ( + !this.#completionPositionIsAllowed( + this.#view.state, + context.pos, + ) + ) { + if (this.#active === active) { + this.#clearCompletionState(); + this.#scheduleCompletionGateClose(capture); + } + return null; + } if ( controller.signal.aborted || this.#active !== active || @@ -607,7 +712,51 @@ export function createSqlEditorInternal< this.#clearCompletionState(); }; + readonly invalidateCatalog = (): SqlRevision | null => { + if (this.#destroyed) return null; + try { + return this.#session.invalidateCatalog(); + } catch { + return null; + } + }; + + readonly statementBoundariesIntersecting = ( + request: SqlStatementBoundariesIntersectingRequest, + ): SqlStatementBoundariesIntersectingResult | null => { + if (this.#destroyed) return null; + try { + return this.#session.statementBoundariesIntersecting(request); + } catch { + return null; + } + }; + + readonly statementBoundaryAt = ( + request: SqlStatementBoundaryAtRequest, + ): SqlStatementBoundaryAtResult | null => { + if (this.#destroyed) return null; + try { + return this.#session.statementBoundaryAt(request); + } catch { + return null; + } + }; + readonly update = (update: ViewUpdate): void => { + const completionPositionAllowed = + this.#completionPositionIsAllowed( + update.state, + update.state.selection.main.head, + ); + const completionPositionBecameDenied = + this.#completionPositionAllowed && + !completionPositionAllowed; + this.#completionPositionAllowed = completionPositionAllowed; + const completionPositionDenied = !completionPositionAllowed; + if (completionPositionDenied) { + this.#clearCompletionState(); + } let contextChanged = false; let regionsChanged = false; for (const transaction of update.transactions) { @@ -691,6 +840,12 @@ export function createSqlEditorInternal< this.#clearCompletionState(); } const nextCompletionStatus = completionStatus(update.state); + if ( + completionPositionBecameDenied && + nextCompletionStatus !== null + ) { + this.#scheduleCompletionGateClose(this.#capture()); + } const nextSelectedCompletion = selectedCompletion(update.state); const nextSelectedCompletionIndex = selectedCompletionIndex( update.state, @@ -744,6 +899,23 @@ export function createSqlEditorInternal< }, }]), ); + const statementGutter = options.statementGutter === false || + options.statementGutter === undefined + ? [] + : createSqlStatementGutter(options.statementGutter, { + boundariesIntersecting: (view, range) => + view.plugin(plugin)?.statementBoundariesIntersecting(range) ?? + null, + boundaryAt: (view, position, affinity) => + view.plugin(plugin)?.statementBoundaryAt({ + affinity, + position, + }) ?? null, + inputKeys: (view) => [ + view.state.field(contextField), + view.state.field(embeddedRegionsField), + ], + }); return Object.freeze({ contextEffect, embeddedRegionsEffect, @@ -752,11 +924,25 @@ export function createSqlEditorInternal< embeddedRegionsField, plugin, escapeKeymap, + statementGutter, autocompletion({ ...autocompleteOptions, override: [completionSource, ...externalSources], }), ], + invalidateCatalog: (view: EditorView): SqlRevision | null => + view.plugin(plugin)?.invalidateCatalog() ?? null, + statementBoundariesIntersecting: ( + view: EditorView, + request: SqlStatementBoundariesIntersectingRequest, + ) => + view.plugin(plugin)?.statementBoundariesIntersecting(request) ?? + null, + statementBoundaryAt: ( + view: EditorView, + request: SqlStatementBoundaryAtRequest, + ) => + view.plugin(plugin)?.statementBoundaryAt(request) ?? null, setContext: ( view: EditorView, context: SqlContextInput, diff --git a/src/vnext/codemirror/statement-gutter.ts b/src/vnext/codemirror/statement-gutter.ts new file mode 100644 index 0000000..392adee --- /dev/null +++ b/src/vnext/codemirror/statement-gutter.ts @@ -0,0 +1,238 @@ +import { + RangeSet, + type Extension, + type Text, +} from "@codemirror/state"; +import { + EditorView, + GutterMarker, + gutter, +} from "@codemirror/view"; +import type { + SqlStatementBoundariesIntersectingResult, + SqlStatementBoundaryAtResult, + SqlTextRange, +} from "../index.js"; + +export interface SqlEditorStatementGutterOptions { + readonly hideWhenNotFocused?: boolean; + readonly showInactive?: boolean; +} + +export interface SqlStatementGutterAccess { + readonly boundaryAt: ( + view: EditorView, + position: number, + affinity: "left" | "right", + ) => SqlStatementBoundaryAtResult | null; + readonly boundariesIntersecting: ( + view: EditorView, + range: SqlTextRange, + ) => SqlStatementBoundariesIntersectingResult | null; + readonly inputKeys: ( + view: EditorView, + ) => readonly [unknown, unknown]; +} + +class SqlStatementGutterMarker extends GutterMarker { + constructor(readonly active: boolean) { + super(); + } + + override eq(other: SqlStatementGutterMarker): boolean { + return this.active === other.active; + } + + override toDOM(view: EditorView): Node { + const marker = view.dom.ownerDocument.createElement("div"); + marker.className = this.active + ? "cm-sql-statement-marker cm-sql-statement-marker-active" + : "cm-sql-statement-marker cm-sql-statement-marker-inactive"; + return marker; + } +} + +const activeMarker = new SqlStatementGutterMarker(true); +const inactiveMarker = new SqlStatementGutterMarker(false); +const emptyMarkers: RangeSet = RangeSet.empty; + +interface SqlStatementGutterSnapshot { + readonly contextKey: unknown; + readonly document: Text; + readonly embeddedRegionsKey: unknown; + readonly focused: boolean; + readonly markers: RangeSet; + readonly selectionHead: number; + readonly viewportFrom: number; + readonly viewportTo: number; +} + +function sameRange( + left: SqlTextRange, + right: SqlTextRange, +): boolean { + return left.from === right.from && left.to === right.to; +} + +function exactCode( + result: SqlStatementBoundaryAtResult | null, +): SqlTextRange | null { + const boundary = result?.boundary; + return boundary?.boundaryQuality === "exact" && + boundary.hasCode + ? boundary.code + : null; +} + +function currentCodeRange( + view: EditorView, + access: SqlStatementGutterAccess, +): SqlTextRange | null { + const position = view.state.selection.main.head; + const right = access.boundaryAt(view, position, "right"); + if ( + right?.boundary.boundaryQuality === "opaque" + ) { + return null; + } + const rightCode = exactCode(right); + if (rightCode !== null) return rightCode; + if ( + right?.boundary.boundaryQuality !== "exact" || + right.boundary.hasCode + ) { + return null; + } + return exactCode( + access.boundaryAt(view, position, "left"), + ); +} + +function buildMarkers( + view: EditorView, + options: SqlEditorStatementGutterOptions, + access: SqlStatementGutterAccess, +): RangeSet { + if ( + options.hideWhenNotFocused === true && !view.hasFocus + ) { + return emptyMarkers; + } + const { from, to } = view.viewport; + if (from === to) return emptyMarkers; + const visible = access.boundariesIntersecting(view, { from, to }); + if (visible === null) return emptyMarkers; + const current = currentCodeRange(view, access); + const lineMarkers = new Map(); + for (const boundary of visible.boundaries) { + if ( + boundary.boundaryQuality !== "exact" || + !boundary.hasCode + ) { + continue; + } + const codeFrom = Math.max(from, boundary.code.from); + const codeTo = Math.min(to, boundary.code.to); + if (codeFrom >= codeTo) continue; + const active = current !== null && + sameRange(boundary.code, current); + let line = view.state.doc.lineAt(codeFrom); + while (line.from < codeTo) { + if (active || options.showInactive !== false) { + lineMarkers.set( + line.from, + active || lineMarkers.get(line.from) === true, + ); + } + if (line.to >= codeTo || line.to === view.state.doc.length) { + break; + } + line = view.state.doc.line(line.number + 1); + } + } + return RangeSet.of( + Array.from(lineMarkers, ([position, active]) => + (active ? activeMarker : inactiveMarker).range(position) + ), + true, + ); +} + +function snapshotMatches( + snapshot: SqlStatementGutterSnapshot, + view: EditorView, + contextKey: unknown, + embeddedRegionsKey: unknown, +): boolean { + return snapshot.document === view.state.doc && + snapshot.selectionHead === view.state.selection.main.head && + snapshot.viewportFrom === view.viewport.from && + snapshot.viewportTo === view.viewport.to && + snapshot.focused === view.hasFocus && + snapshot.contextKey === contextKey && + snapshot.embeddedRegionsKey === embeddedRegionsKey; +} + +export function createSqlStatementGutter( + options: SqlEditorStatementGutterOptions, + access: SqlStatementGutterAccess, +): Extension { + const snapshots = new WeakMap< + EditorView, + SqlStatementGutterSnapshot + >(); + return [ + gutter({ + class: "cm-sql-statement-gutter", + markers: (view) => { + const [contextKey, embeddedRegionsKey] = + access.inputKeys(view); + const previous = snapshots.get(view); + if ( + previous !== undefined && + snapshotMatches( + previous, + view, + contextKey, + embeddedRegionsKey, + ) + ) { + return previous.markers; + } + const markers = buildMarkers(view, options, access); + snapshots.set(view, { + contextKey, + document: view.state.doc, + embeddedRegionsKey, + focused: view.hasFocus, + markers, + selectionHead: view.state.selection.main.head, + viewportFrom: view.viewport.from, + viewportTo: view.viewport.to, + }); + return markers; + }, + }), + EditorView.baseTheme({ + ".cm-sql-statement-gutter": { + minWidth: "3px", + width: "3px", + }, + ".cm-sql-statement-gutter .cm-gutterElement": { + margin: "0", + padding: "0", + width: "3px", + }, + ".cm-sql-statement-marker": { + backgroundColor: "var(--cm-sql-statement-color, #3b82f6)", + borderRadius: "1px", + display: "block", + height: "100%", + width: "3px", + }, + ".cm-sql-statement-marker-inactive": { + opacity: "var(--cm-sql-statement-inactive-opacity, 0.3)", + }, + }), + ]; +} diff --git a/src/vnext/column-catalog-batch-coordinator.ts b/src/vnext/column-catalog-batch-coordinator.ts new file mode 100644 index 0000000..38ba9d7 --- /dev/null +++ b/src/vnext/column-catalog-batch-coordinator.ts @@ -0,0 +1,566 @@ +import { + captureSqlColumnCatalogProvider, + createSqlColumnCatalogBatchRequest, + decodeSqlColumnCatalogBatchResponse, + MAX_COLUMN_DIALECT_ID_LENGTH, + MAX_COLUMN_SCOPE_LENGTH, + resolveSqlColumnCatalogProvider, + type CapturedSqlColumnCatalogProvider, + type SqlCapturedColumnCatalogProviderContext, +} from "./column-catalog-boundary.js"; +import type { + SqlColumnCatalogBatchRequest, + SqlColumnCatalogRelationReference, + SqlColumnCatalogRelationResult, +} from "./column-catalog-types.js"; +import type { + SqlCatalogEpoch, +} from "./relation-completion-types.js"; +import type { SqlIdentifierPath } from "./types.js"; + +export const DEFAULT_COLUMN_CACHE_ENTRIES = 1_024; +export const MAX_COLUMN_CACHE_ENTRIES = 8_192; + +export interface SqlColumnCatalogOwnerOptions { + readonly dialectId: string; + readonly scope: string; +} + +export interface SqlColumnCatalogBatchInput { + readonly expectedEpoch: SqlCatalogEpoch | null; + readonly relations: readonly SqlColumnCatalogRelationReference[]; + readonly searchPaths: readonly SqlIdentifierPath[]; +} + +export type SqlColumnCatalogBatchOutcome = + | { + readonly epoch: SqlCatalogEpoch; + readonly providerId: string; + readonly relations: readonly SqlColumnCatalogRelationResult[]; + readonly scope: string; + readonly status: "usable"; + } + | { + readonly status: "cancelled"; + } + | { + readonly status: "superseded"; + } + | { + readonly reason: + | "disposed" + | "invalid-request" + | "malformed-response" + | "provider-failed"; + readonly status: "unavailable"; + }; + +export interface SqlColumnCatalogBatchTicket { + readonly cancel: (this: void) => void; + readonly result: Promise; +} + +export interface SqlColumnCatalogBatchOwner { + readonly dispose: (this: void) => void; + readonly request: ( + this: void, + input: SqlColumnCatalogBatchInput, + ) => SqlColumnCatalogBatchTicket; +} + +export type SqlColumnCatalogBatchOwnerResult = + | { + readonly owner: SqlColumnCatalogBatchOwner; + readonly status: "prepared"; + } + | { + readonly reason: "disposed" | "invalid-options"; + readonly status: "unavailable"; + }; + +export interface SqlColumnCatalogBatchCoordinator { + readonly dispose: (this: void) => void; + readonly prepareOwner: ( + this: void, + options: SqlColumnCatalogOwnerOptions, + ) => SqlColumnCatalogBatchOwnerResult; + readonly providerId: string; +} + +export type SqlColumnCatalogBatchCoordinatorResult = + | { + readonly coordinator: SqlColumnCatalogBatchCoordinator; + readonly status: "created"; + } + | { + readonly reason: "invalid-options" | "invalid-provider"; + readonly status: "unavailable"; + }; + +interface CoordinatorState { + readonly map: Map; + off: boolean; + readonly id: string; + readonly limit: number; + readonly load: SqlCapturedColumnCatalogProviderContext["loadColumns"]; + readonly pool: Set; + readonly wire: CapturedSqlColumnCatalogProvider; +} + +interface OwnerState { + readonly lang: string; + gen: number; + job: ConsumerState | null; + off: boolean; + rev: SqlCatalogEpoch | null; + root: CoordinatorState | null; + readonly scope: string; +} + +interface ConsumerState { + readonly abort: AbortController; + done: boolean; + end: ((value: SqlColumnCatalogBatchOutcome) => void) | null; + readonly host: OwnerState; +} + +type ReadyRelation = Extract< + SqlColumnCatalogRelationResult, + { readonly status: "ready" } +>; + +const CANCELLED: SqlColumnCatalogBatchOutcome = + Object.freeze({ status: "cancelled" }); +const SUPERSEDED: SqlColumnCatalogBatchOutcome = + Object.freeze({ status: "superseded" }); + +function unavailable( + reason: Extract< + SqlColumnCatalogBatchOutcome, + { readonly status: "unavailable" } + >["reason"], +): SqlColumnCatalogBatchOutcome { + return Object.freeze({ reason, status: "unavailable" }); +} + +function dataProperty( + value: unknown, + key: string, +): { readonly found: true; readonly value: unknown } | null { + if (value === null || typeof value !== "object") return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, key); + } catch { + return null; + } + return descriptor && "value" in descriptor + ? { found: true, value: descriptor.value } + : null; +} + +function boundedString(value: unknown, maximum: number): string | null { + return typeof value === "string" && + value.length > 0 && + value.length <= maximum + ? value + : null; +} + +function cacheKey( + request: SqlColumnCatalogBatchRequest, + reference: SqlColumnCatalogRelationReference, + epoch: SqlCatalogEpoch, +): string { + const segment = (value: string): string => + `${value.length}:${value}`; + const path = reference.path.map((component) => + segment(`${component.quoted ? "q" : "u"}${component.value}`) + ).join(""); + const searchPaths = request.searchPaths.map((searchPath) => + segment(searchPath.map((component) => + segment(`${component.quoted ? "q" : "u"}${component.value}`) + ).join("")) + ).join(""); + return [ + segment(request.scope), + segment(request.dialectId), + segment(String(epoch.generation)), + segment(epoch.token), + segment(path), + segment(searchPaths), + ].join(""); +} + +function cacheGet( + state: CoordinatorState, + key: string, +): ReadyRelation | null { + const value = state.map.get(key); + if (!value) return null; + state.map.delete(key); + state.map.set(key, value); + return value; +} + +function cacheSet( + state: CoordinatorState, + key: string, + value: ReadyRelation, +): void { + state.map.delete(key); + state.map.set(key, value); + while (state.map.size > state.limit) { + for (const oldest of state.map.keys()) { + state.map.delete(oldest); + break; + } + } +} + +function withRequestKey( + relation: ReadyRelation, + requestKey: string, +): ReadyRelation { + return relation.requestKey === requestKey + ? relation + : Object.freeze({ ...relation, requestKey }); +} + +function settle( + consumer: ConsumerState, + outcome: SqlColumnCatalogBatchOutcome, +): void { + if (consumer.host.job === consumer) { + consumer.host.job = null; + } + if (consumer.done) return; + consumer.done = true; + const resolve = consumer.end; + consumer.end = null; + resolve?.(outcome); +} + +function cancelConsumer( + consumer: ConsumerState, + outcome: SqlColumnCatalogBatchOutcome, +): void { + if (consumer.done) return; + consumer.abort.abort(); + settle(consumer, outcome); +} + +function makeSettledTicket( + outcome: SqlColumnCatalogBatchOutcome, +): SqlColumnCatalogBatchTicket { + return Object.freeze({ + cancel: (): void => {}, + result: Promise.resolve(outcome), + }); +} + +function combineRelations( + cached: ReadonlyMap, + loaded: readonly SqlColumnCatalogRelationResult[], +): readonly SqlColumnCatalogRelationResult[] { + const output: SqlColumnCatalogRelationResult[] = [...loaded]; + for (const [requestKey, relation] of cached) { + output.push(withRequestKey(relation, requestKey)); + } + output.sort((left, right) => + left.requestKey < right.requestKey + ? -1 + : left.requestKey > right.requestKey + ? 1 + : 0 + ); + return Object.freeze(output); +} + +function startProviderWork( + state: CoordinatorState, + owner: OwnerState, + missingRequest: SqlColumnCatalogBatchRequest, + cached: ReadonlyMap, + consumer: ConsumerState, +): void { + let providerResult: unknown; + try { + providerResult = state.load( + missingRequest, + consumer.abort.signal, + ); + } catch { + settle(consumer, unavailable("provider-failed")); + return; + } + Promise.resolve(providerResult).then( + (value) => { + if (consumer.done) return; + let decoded: ReturnType< + typeof decodeSqlColumnCatalogBatchResponse + >; + try { + decoded = decodeSqlColumnCatalogBatchResponse( + state.wire, + missingRequest, + value, + ); + } catch { + settle(consumer, unavailable("malformed-response")); + return; + } + if (decoded.status === "malformed") { + settle(consumer, unavailable("malformed-response")); + return; + } + owner.rev = decoded.value.epoch; + for (const relation of decoded.value.relations) { + if ( + relation.status !== "ready" || + relation.coverage !== "complete" + ) continue; + const reference = missingRequest.relations.find( + (candidate) => candidate.requestKey === relation.requestKey, + ); + if (reference) { + cacheSet( + state, + cacheKey(missingRequest, reference, decoded.value.epoch), + relation, + ); + } + } + const relations = combineRelations( + cached, + decoded.value.relations, + ); + settle( + consumer, + Object.freeze({ + providerId: state.id, + epoch: decoded.value.epoch, + relations, + scope: owner.scope, + status: "usable", + }), + ); + }, + () => { + if (!consumer.done) { + settle(consumer, unavailable("provider-failed")); + } + }, + ); +} + +function requestColumns( + owner: OwnerState, + input: unknown, +): SqlColumnCatalogBatchTicket { + const state = owner.root; + if (!state || state.off || owner.off) { + return makeSettledTicket(unavailable("disposed")); + } + const expectedEpoch = dataProperty(input, "expectedEpoch"); + const relations = dataProperty(input, "relations"); + const searchPaths = dataProperty(input, "searchPaths"); + if (!expectedEpoch || !relations || !searchPaths) { + return makeSettledTicket(unavailable("invalid-request")); + } + const created = createSqlColumnCatalogBatchRequest({ + dialectId: owner.lang, + expectedEpoch: expectedEpoch.value === null + ? owner.rev + : expectedEpoch.value, + relations: relations.value, + scope: owner.scope, + searchPaths: searchPaths.value, + }); + if (created.status === "malformed") { + return makeSettledTicket(unavailable("invalid-request")); + } + const request = created.value; + owner.gen += 1; + const generation = owner.gen; + if (owner.job) cancelConsumer(owner.job, SUPERSEDED); + if (generation !== owner.gen) { + return makeSettledTicket( + owner.off ? unavailable("disposed") : SUPERSEDED, + ); + } + + const cached = new Map(); + const missing: SqlColumnCatalogRelationReference[] = []; + for (const reference of request.relations) { + const key = request.expectedEpoch === null + ? null + : cacheKey(request, reference, request.expectedEpoch); + const cachedRelation = key === null ? null : cacheGet(state, key); + if (cachedRelation) { + cached.set(reference.requestKey, cachedRelation); + } else { + missing.push(reference); + } + } + if (request.expectedEpoch !== null && missing.length === 0) { + return makeSettledTicket( + Object.freeze({ + epoch: request.expectedEpoch, + providerId: state.id, + relations: combineRelations(cached, []), + scope: owner.scope, + status: "usable", + }), + ); + } + const missingRequest: SqlColumnCatalogBatchRequest = Object.freeze({ + ...request, + relations: Object.freeze(missing), + }); + let resolveResult: ( + value: SqlColumnCatalogBatchOutcome, + ) => void = (): void => {}; + const result = new Promise((resolve) => { + resolveResult = resolve; + }); + const consumer: ConsumerState = { + abort: new AbortController(), + done: false, + end: resolveResult, + host: owner, + }; + owner.job = consumer; + const ticket = Object.freeze({ + cancel: (): void => cancelConsumer(consumer, CANCELLED), + result, + }); + startProviderWork( + state, + owner, + missingRequest, + cached, + consumer, + ); + return ticket; +} + +function disposeOwner(owner: OwnerState): void { + if (owner.off) return; + owner.off = true; + owner.gen += 1; + if (owner.job) { + cancelConsumer(owner.job, unavailable("disposed")); + } + owner.root?.pool.delete(owner); + owner.root = null; +} + +function prepareOwner( + state: CoordinatorState, + input: unknown, +): SqlColumnCatalogBatchOwnerResult { + if (state.off) { + return Object.freeze({ reason: "disposed", status: "unavailable" }); + } + const scopeProperty = dataProperty(input, "scope"); + const dialectProperty = dataProperty(input, "dialectId"); + const scope = boundedString( + scopeProperty?.value, + MAX_COLUMN_SCOPE_LENGTH, + ); + const dialectId = boundedString( + dialectProperty?.value, + MAX_COLUMN_DIALECT_ID_LENGTH, + ); + if (scope === null || dialectId === null) { + return Object.freeze({ + reason: "invalid-options", + status: "unavailable", + }); + } + const owner: OwnerState = { + gen: 0, + job: null, + lang: dialectId, + off: false, + rev: null, + root: state, + scope, + }; + state.pool.add(owner); + return Object.freeze({ + owner: Object.freeze({ + dispose: (): void => disposeOwner(owner), + request: (input_: SqlColumnCatalogBatchInput) => + requestColumns(owner, input_), + }), + status: "prepared", + }); +} + +function disposeCoordinator(state: CoordinatorState): void { + if (state.off) return; + state.off = true; + state.map.clear(); + for (const owner of state.pool) disposeOwner(owner); +} + +export function createSqlColumnCatalogBatchCoordinator( + input: unknown, +): SqlColumnCatalogBatchCoordinatorResult { + const providerProperty = dataProperty(input, "provider"); + if (!providerProperty) { + return Object.freeze({ + reason: "invalid-provider", + status: "unavailable", + }); + } + const captured = captureSqlColumnCatalogProvider( + providerProperty.value, + ); + if (captured.status === "malformed") { + return Object.freeze({ + reason: "invalid-provider", + status: "unavailable", + }); + } + const context = resolveSqlColumnCatalogProvider(captured.value); + if (!context) { + return Object.freeze({ + reason: "invalid-provider", + status: "unavailable", + }); + } + const maximumProperty = dataProperty(input, "maxCacheEntries"); + const maximum = maximumProperty?.value ?? DEFAULT_COLUMN_CACHE_ENTRIES; + if ( + typeof maximum !== "number" || + !Number.isSafeInteger(maximum) || + maximum < 1 || + maximum > MAX_COLUMN_CACHE_ENTRIES + ) { + return Object.freeze({ + reason: "invalid-options", + status: "unavailable", + }); + } + const state: CoordinatorState = { + id: context.id, + limit: maximum, + load: context.loadColumns, + map: new Map(), + off: false, + pool: new Set(), + wire: captured.value, + }; + return Object.freeze({ + coordinator: Object.freeze({ + dispose: (): void => disposeCoordinator(state), + prepareOwner: (options: SqlColumnCatalogOwnerOptions) => + prepareOwner(state, options), + providerId: state.id, + }), + status: "created", + }); +} diff --git a/src/vnext/column-catalog-boundary.ts b/src/vnext/column-catalog-boundary.ts new file mode 100644 index 0000000..3c1f573 --- /dev/null +++ b/src/vnext/column-catalog-boundary.ts @@ -0,0 +1,694 @@ +import type { SqlCatalogEpoch } from "./relation-completion-types.js"; +import type { + SqlColumnCatalogBatchRequest, + SqlColumnCatalogBatchResponse, + SqlColumnCatalogColumn, + SqlColumnCatalogRelationReference, + SqlColumnCatalogRelationResult, + SqlColumnCatalogResolvedColumn, +} from "./column-catalog-types.js"; +import { + isDataArray, + type SqlIdentifierComponent, + type SqlIdentifierPath, +} from "./types.js"; + +export const MAX_COLUMN_PROVIDER_ID_LENGTH = 256; +export const MAX_COLUMN_SCOPE_LENGTH = 512; +export const MAX_COLUMN_DIALECT_ID_LENGTH = 128; +export const MAX_COLUMN_EPOCH_TOKEN_LENGTH = 256; +export const MAX_COLUMN_ENTITY_ID_LENGTH = 256; +export const MAX_COLUMN_NAME_LENGTH = 256; +export const MAX_COLUMN_INSERT_TEXT_LENGTH = 1_024; +export const MAX_COLUMN_TYPE_LENGTH = 512; +export const MAX_COLUMN_DETAIL_LENGTH = 1_024; +export const MAX_COLUMN_BATCH_RELATIONS = 64; +export const MAX_COLUMN_RELATION_PATH_COMPONENTS = 32; +export const MAX_COLUMN_SEARCH_PATHS = 32; +export const MAX_COLUMN_SEARCH_PATH_COMPONENTS = 8; +export const MAX_COLUMNS_PER_RELATION = 256; +export const MAX_COLUMNS_PER_BATCH = 4_096; + +const capturedColumnProviders = new WeakMap< + object, + { + readonly id: string; + readonly loadColumns: Function; + } +>(); +const capturedColumnProviderBrand: unique symbol = Symbol( + "CapturedSqlColumnCatalogProvider", +); + +export interface CapturedSqlColumnCatalogProvider { + readonly [capturedColumnProviderBrand]: + "CapturedSqlColumnCatalogProvider"; +} + +export interface SqlCapturedColumnCatalogProviderContext { + readonly id: string; + readonly loadColumns: ( + this: void, + request: SqlColumnCatalogBatchRequest, + signal: AbortSignal, + ) => unknown; +} + +export type SqlColumnBoundaryFailureReason = + | "duplicate-column-entity-id" + | "duplicate-request-key" + | "invalid-shape" + | "resource-limit" + | "unexpected-relation"; + +export type SqlColumnBoundaryResult = + | { + readonly status: "accepted"; + readonly value: Value; + } + | { + readonly reason: SqlColumnBoundaryFailureReason; + readonly status: "malformed"; + }; + +interface DataRecord { + readonly fields: ReadonlyMap; +} + +const FAILURE_CODES = new Set([ + "authentication", + "authorization", + "invalid-configuration", + "rate-limited", + "unavailable", + "unknown", +]); +const RETRY_POLICIES = new Set([ + "after-invalidation", + "never", + "next-request", +]); + +type FailureCode = Extract< + SqlColumnCatalogRelationResult, + { readonly status: "failed" } +>["code"]; +type RetryPolicy = Extract< + SqlColumnCatalogRelationResult, + { readonly status: "failed" } +>["retry"]; + +function isFailureCode(value: unknown): value is FailureCode { + return typeof value === "string" && FAILURE_CODES.has(value); +} + +function isRetryPolicy(value: unknown): value is RetryPolicy { + return typeof value === "string" && RETRY_POLICIES.has(value); +} + +function accepted( + value: Value, +): SqlColumnBoundaryResult { + return Object.freeze({ status: "accepted", value }); +} + +function malformed( + reason: SqlColumnBoundaryFailureReason, +): SqlColumnBoundaryResult { + return Object.freeze({ reason, status: "malformed" }); +} + +function readRecord( + value: unknown, + allowedKeys: ReadonlySet, +): DataRecord | null { + if (value === null || typeof value !== "object") { + return null; + } + let keys: readonly PropertyKey[]; + try { + keys = Reflect.ownKeys(value); + } catch { + return null; + } + const fields = new Map(); + for (const key of keys) { + if (typeof key !== "string" || !allowedKeys.has(key)) return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, key); + } catch { + return null; + } + if (!descriptor || !("value" in descriptor)) return null; + fields.set(key, descriptor.value); + } + return { fields }; +} + +function required(record: DataRecord, key: string): unknown { + return record.fields.has(key) ? record.fields.get(key) : undefined; +} + +function boundedString(value: unknown, maximum: number): string | null { + return typeof value === "string" && + value.length > 0 && + value.length <= maximum + ? value + : null; +} + +function optionalBoundedString( + value: unknown, + maximum: number, +): string | undefined | null { + return value === undefined + ? undefined + : boundedString(value, maximum); +} + +function decodeEpoch(value: unknown): SqlCatalogEpoch | null { + const record = readRecord( + value, + new Set(["generation", "token"]), + ); + if (!record) return null; + const generation = required(record, "generation"); + const token = boundedString( + required(record, "token"), + MAX_COLUMN_EPOCH_TOKEN_LENGTH, + ); + if ( + typeof generation !== "number" || + !Number.isSafeInteger(generation) || + generation < 0 || + token === null + ) { + return null; + } + return Object.freeze({ generation, token }); +} + +function decodeExpectedEpoch( + value: unknown, +): SqlCatalogEpoch | null | undefined { + return value === null ? null : decodeEpoch(value) ?? undefined; +} + +function readArrayElement( + values: readonly unknown[], + index: number, +): { readonly found: true; readonly value: unknown } | null { + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(values, index); + } catch { + return null; + } + return descriptor && "value" in descriptor + ? { found: true, value: descriptor.value } + : null; +} + +function readArrayLength(value: unknown, maximum: number): number | null { + if (!isDataArray(value)) return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, "length"); + } catch { + return null; + } + if ( + !descriptor || + !("value" in descriptor) || + typeof descriptor.value !== "number" || + !Number.isSafeInteger(descriptor.value) || + descriptor.value < 0 || + descriptor.value > maximum + ) { + return null; + } + return descriptor.value; +} + +function decodeIdentifierPath( + value: unknown, + maximumComponents: number, +): SqlIdentifierPath | null { + const length = readArrayLength(value, maximumComponents); + if (length === null || length === 0 || !Array.isArray(value)) return null; + const output: SqlIdentifierComponent[] = []; + for (let index = 0; index < length; index += 1) { + const item = readArrayElement(value, index); + if (!item) return null; + const record = readRecord( + item.value, + new Set(["quoted", "value"]), + ); + if (!record) return null; + const identifier = boundedString( + required(record, "value"), + MAX_COLUMN_NAME_LENGTH, + ); + const quoted = required(record, "quoted"); + if (identifier === null || typeof quoted !== "boolean") return null; + output.push(Object.freeze({ quoted, value: identifier })); + } + return Object.freeze(output); +} + +function decodeSearchPaths(value: unknown): readonly SqlIdentifierPath[] | null { + const length = readArrayLength(value, MAX_COLUMN_SEARCH_PATHS); + if (length === null || !Array.isArray(value)) return null; + const paths: SqlIdentifierPath[] = []; + for (let index = 0; index < length; index += 1) { + const item = readArrayElement(value, index); + if (!item) return null; + const path = decodeIdentifierPath( + item.value, + MAX_COLUMN_SEARCH_PATH_COMPONENTS, + ); + if (!path) return null; + paths.push(path); + } + return Object.freeze(paths); +} + +function decodeRelationReference( + value: unknown, +): SqlColumnCatalogRelationReference | null { + const record = readRecord( + value, + new Set(["path", "requestKey"]), + ); + if (!record) return null; + const requestKey = boundedString( + required(record, "requestKey"), + MAX_COLUMN_ENTITY_ID_LENGTH, + ); + const path = decodeIdentifierPath( + required(record, "path"), + MAX_COLUMN_RELATION_PATH_COMPONENTS, + ); + if (requestKey === null || !path) return null; + return Object.freeze({ + path, + requestKey, + }); +} + +function sameEpoch(left: SqlCatalogEpoch, right: SqlCatalogEpoch): boolean { + return left.generation === right.generation && left.token === right.token; +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +export function captureSqlColumnCatalogProvider( + provider: unknown, +): SqlColumnBoundaryResult { + const record = readRecord(provider, new Set(["id", "loadColumns"])); + if (!record) return malformed("invalid-shape"); + const id = boundedString( + required(record, "id"), + MAX_COLUMN_PROVIDER_ID_LENGTH, + ); + const loadColumns = required(record, "loadColumns"); + if (id === null || typeof loadColumns !== "function") { + return malformed("invalid-shape"); + } + const capturedValue: CapturedSqlColumnCatalogProvider = { + [capturedColumnProviderBrand]: "CapturedSqlColumnCatalogProvider", + }; + const captured = Object.freeze(capturedValue); + capturedColumnProviders.set(captured, { id, loadColumns }); + return accepted(captured); +} + +export function resolveSqlColumnCatalogProvider( + provider: unknown, +): SqlCapturedColumnCatalogProviderContext | null { + if (provider === null || typeof provider !== "object") return null; + const captured = capturedColumnProviders.get(provider); + if (!captured) return null; + return Object.freeze({ + id: captured.id, + loadColumns: ( + request: SqlColumnCatalogBatchRequest, + signal: AbortSignal, + ): unknown => + Reflect.apply( + captured.loadColumns, + undefined, + [request, signal], + ), + }); +} + +export function createSqlColumnCatalogBatchRequest( + input: unknown, +): SqlColumnBoundaryResult { + const record = readRecord( + input, + new Set([ + "dialectId", + "expectedEpoch", + "relations", + "scope", + "searchPaths", + ]), + ); + if (!record) return malformed("invalid-shape"); + const scope = boundedString( + required(record, "scope"), + MAX_COLUMN_SCOPE_LENGTH, + ); + const dialectId = boundedString( + required(record, "dialectId"), + MAX_COLUMN_DIALECT_ID_LENGTH, + ); + const expectedEpoch = decodeExpectedEpoch( + required(record, "expectedEpoch"), + ); + const relationsValue = required(record, "relations"); + const searchPaths = decodeSearchPaths(required(record, "searchPaths")); + const relationCount = readArrayLength( + relationsValue, + MAX_COLUMN_BATCH_RELATIONS, + ); + if ( + scope === null || + dialectId === null || + expectedEpoch === undefined || + relationCount === null || + !Array.isArray(relationsValue) || + searchPaths === null + ) { + return malformed("invalid-shape"); + } + const keys = new Set(); + const relations: SqlColumnCatalogRelationReference[] = []; + for (let index = 0; index < relationCount; index += 1) { + const item = readArrayElement(relationsValue, index); + if (!item) return malformed("invalid-shape"); + const reference = decodeRelationReference(item.value); + if (!reference) return malformed("invalid-shape"); + if (keys.has(reference.requestKey)) { + return malformed("duplicate-request-key"); + } + keys.add(reference.requestKey); + relations.push(reference); + } + if (relations.length === 0) return malformed("invalid-shape"); + relations.sort((left, right) => + compareText(left.requestKey, right.requestKey) + ); + return accepted(Object.freeze({ + dialectId, + expectedEpoch, + relations: Object.freeze(relations), + searchPaths, + scope, + })); +} + +function decodeColumn( + value: unknown, + providerId: string, + request: SqlColumnCatalogBatchRequest, + epoch: SqlCatalogEpoch, + relationEntityId: string, +): SqlColumnCatalogResolvedColumn | null { + const record = readRecord( + value, + new Set([ + "columnEntityId", + "dataType", + "detail", + "identifier", + "insertText", + "ordinal", + ]), + ); + if (!record) return null; + const columnEntityId = boundedString( + required(record, "columnEntityId"), + MAX_COLUMN_ENTITY_ID_LENGTH, + ); + const identifierRecord = readRecord( + required(record, "identifier"), + new Set(["quoted", "value"]), + ); + const identifierValue = identifierRecord + ? boundedString( + required(identifierRecord, "value"), + MAX_COLUMN_NAME_LENGTH, + ) + : null; + const quoted = identifierRecord + ? required(identifierRecord, "quoted") + : null; + const insertText = boundedString( + required(record, "insertText"), + MAX_COLUMN_INSERT_TEXT_LENGTH, + ); + const ordinal = required(record, "ordinal"); + const dataType = optionalBoundedString( + required(record, "dataType"), + MAX_COLUMN_TYPE_LENGTH, + ); + const detail = optionalBoundedString( + required(record, "detail"), + MAX_COLUMN_DETAIL_LENGTH, + ); + if ( + columnEntityId === null || + identifierValue === null || + typeof quoted !== "boolean" || + insertText === null || + dataType === null || + detail === null || + typeof ordinal !== "number" || + !Number.isSafeInteger(ordinal) || + ordinal < 0 + ) { + return null; + } + const column: SqlColumnCatalogColumn = { + columnEntityId, + identifier: Object.freeze({ + quoted, + value: identifierValue, + }), + insertText, + ordinal, + ...(dataType === undefined ? {} : { dataType }), + ...(detail === undefined ? {} : { detail }), + }; + return Object.freeze({ + ...column, + provenance: Object.freeze({ + columnEntityId, + epoch, + providerId, + relationEntityId, + scope: request.scope, + }), + }); +} + +function sameColumn( + left: SqlColumnCatalogResolvedColumn, + right: SqlColumnCatalogResolvedColumn, +): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +function decodeReadyRelation( + record: DataRecord, + providerId: string, + request: SqlColumnCatalogBatchRequest, + epoch: SqlCatalogEpoch, + requestKey: string, + relationEntityId: string, +): SqlColumnBoundaryResult { + const coverage = required(record, "coverage"); + const columnsValue = required(record, "columns"); + const columnCount = readArrayLength( + columnsValue, + MAX_COLUMNS_PER_RELATION, + ); + if ( + (coverage !== "complete" && coverage !== "partial") || + columnCount === null || + !isDataArray(columnsValue) + ) { + return malformed("invalid-shape"); + } + const byId = new Map(); + for (let index = 0; index < columnCount; index += 1) { + const item = readArrayElement(columnsValue, index); + if (!item) return malformed("invalid-shape"); + const column = decodeColumn( + item.value, + providerId, + request, + epoch, + relationEntityId, + ); + if (!column) return malformed("invalid-shape"); + const previous = byId.get(column.columnEntityId); + if (previous && !sameColumn(previous, column)) { + return malformed("duplicate-column-entity-id"); + } + byId.set(column.columnEntityId, column); + } + const columns = [...byId.values()].sort((left, right) => + left.ordinal - right.ordinal || + compareText(left.identifier.value, right.identifier.value) || + compareText(left.columnEntityId, right.columnEntityId) + ); + return accepted(Object.freeze({ + columns: Object.freeze(columns), + coverage, + relationEntityId, + requestKey, + status: "ready", + })); +} + +function decodeRelation( + value: unknown, + providerId: string, + request: SqlColumnCatalogBatchRequest, + epoch: SqlCatalogEpoch, +): SqlColumnBoundaryResult { + const record = readRecord( + value, + new Set([ + "code", + "columns", + "coverage", + "relationEntityId", + "requestKey", + "retry", + "status", + ]), + ); + if (!record) return malformed("invalid-shape"); + const requestKey = boundedString( + required(record, "requestKey"), + MAX_COLUMN_ENTITY_ID_LENGTH, + ); + const status = required(record, "status"); + if (requestKey === null) return malformed("invalid-shape"); + if (status === "ready") { + const relationEntityId = boundedString( + required(record, "relationEntityId"), + MAX_COLUMN_ENTITY_ID_LENGTH, + ); + if ( + relationEntityId === null || + record.fields.size !== 5 + ) return malformed("invalid-shape"); + return decodeReadyRelation( + record, + providerId, + request, + epoch, + requestKey, + relationEntityId, + ); + } + if (status === "loading") { + if (record.fields.size !== 2) return malformed("invalid-shape"); + return accepted(Object.freeze({ + requestKey, + status, + })); + } + const code = required(record, "code"); + const retry = required(record, "retry"); + if ( + status !== "failed" || + record.fields.size !== 4 || + !isFailureCode(code) || + !isRetryPolicy(retry) + ) { + return malformed("invalid-shape"); + } + return accepted(Object.freeze({ + code, + requestKey, + retry, + status, + })); +} + +export function decodeSqlColumnCatalogBatchResponse( + provider: CapturedSqlColumnCatalogProvider, + request: SqlColumnCatalogBatchRequest, + value: unknown, +): SqlColumnBoundaryResult { + const context = resolveSqlColumnCatalogProvider(provider); + const record = readRecord(value, new Set(["epoch", "relations"])); + if (!context || !record) return malformed("invalid-shape"); + const epoch = decodeEpoch(required(record, "epoch")); + const relationsValue = required(record, "relations"); + const relationCount = readArrayLength( + relationsValue, + MAX_COLUMN_BATCH_RELATIONS, + ); + if ( + epoch === null || + (request.expectedEpoch !== null && + !sameEpoch(epoch, request.expectedEpoch)) || + relationCount === null || + !Array.isArray(relationsValue) + ) { + return malformed("invalid-shape"); + } + if (relationCount > request.relations.length) { + return malformed("resource-limit"); + } + const requested = new Set( + request.relations.map((relation) => relation.requestKey), + ); + const byId = new Map(); + let columnCount = 0; + for (let index = 0; index < relationCount; index += 1) { + const item = readArrayElement(relationsValue, index); + if (!item) return malformed("invalid-shape"); + const decoded = decodeRelation( + item.value, + context.id, + request, + epoch, + ); + if (decoded.status === "malformed") return decoded; + const relation = decoded.value; + if (!requested.has(relation.requestKey)) { + return malformed("unexpected-relation"); + } + if (byId.has(relation.requestKey)) { + return malformed("duplicate-request-key"); + } + if (relation.status === "ready") { + columnCount += relation.columns.length; + if (columnCount > MAX_COLUMNS_PER_BATCH) { + return malformed("resource-limit"); + } + } + byId.set(relation.requestKey, relation); + } + if (byId.size !== requested.size) return malformed("invalid-shape"); + return accepted(Object.freeze({ + epoch, + relations: Object.freeze( + [...byId.values()].sort((left, right) => + compareText(left.requestKey, right.requestKey) + ), + ), + })); +} diff --git a/src/vnext/column-catalog-types.ts b/src/vnext/column-catalog-types.ts new file mode 100644 index 0000000..f2a4690 --- /dev/null +++ b/src/vnext/column-catalog-types.ts @@ -0,0 +1,100 @@ +import type { SqlCatalogEpoch } from "./relation-completion-types.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, +} from "./types.js"; + +export type SqlColumnCatalogCoverage = "complete" | "partial"; + +export interface SqlColumnCatalogRelationReference { + readonly path: SqlIdentifierPath; + readonly requestKey: string; +} + +export interface SqlColumnCatalogBatchRequest { + readonly dialectId: string; + readonly expectedEpoch: SqlCatalogEpoch | null; + readonly relations: readonly SqlColumnCatalogRelationReference[]; + readonly searchPaths: readonly SqlIdentifierPath[]; + readonly scope: string; +} + +export interface SqlColumnCatalogColumn { + readonly columnEntityId: string; + readonly dataType?: string; + readonly detail?: string; + readonly identifier: SqlIdentifierComponent; + readonly insertText: string; + readonly ordinal: number; +} + +export interface SqlColumnCatalogProvenance { + readonly columnEntityId: string; + readonly epoch: SqlCatalogEpoch; + readonly providerId: string; + readonly relationEntityId: string; + readonly scope: string; +} + +export interface SqlColumnCatalogResolvedColumn + extends SqlColumnCatalogColumn { + readonly provenance: SqlColumnCatalogProvenance; +} + +export type SqlColumnCatalogRelationResult = + | { + readonly columns: readonly SqlColumnCatalogResolvedColumn[]; + readonly coverage: SqlColumnCatalogCoverage; + readonly relationEntityId: string; + readonly requestKey: string; + readonly status: "ready"; + } + | { + readonly requestKey: string; + readonly status: "loading"; + } + | { + readonly code: + | "authentication" + | "authorization" + | "invalid-configuration" + | "rate-limited" + | "unavailable" + | "unknown"; + readonly requestKey: string; + readonly retry: "after-invalidation" | "never" | "next-request"; + readonly status: "failed"; + }; + +export interface SqlColumnCatalogBatchResponse { + readonly epoch: SqlCatalogEpoch; + readonly relations: readonly SqlColumnCatalogRelationResult[]; +} + +export type SqlColumnCatalogProviderRelationResult = + | { + readonly columns: readonly SqlColumnCatalogColumn[]; + readonly coverage: SqlColumnCatalogCoverage; + readonly relationEntityId: string; + readonly requestKey: string; + readonly status: "ready"; + } + | Extract< + SqlColumnCatalogRelationResult, + { readonly status: "loading" | "failed" } + >; + +export interface SqlColumnCatalogProviderResponse { + readonly epoch: SqlCatalogEpoch; + readonly relations: + readonly SqlColumnCatalogProviderRelationResult[]; +} + +export interface SqlColumnCatalogProvider { + readonly id: string; + readonly loadColumns: ( + this: void, + request: SqlColumnCatalogBatchRequest, + signal: AbortSignal, + ) => Promise; +} diff --git a/src/vnext/column-completion.ts b/src/vnext/column-completion.ts new file mode 100644 index 0000000..8ea49e9 --- /dev/null +++ b/src/vnext/column-completion.ts @@ -0,0 +1,350 @@ +import type { + SqlColumnCatalogBatchOutcome, +} from "./column-catalog-batch-coordinator.js"; +import { + MAX_COLUMN_BATCH_RELATIONS, +} from "./column-catalog-boundary.js"; +import type { + SqlColumnCatalogRelationReference, + SqlColumnCatalogRelationResult, +} from "./column-catalog-types.js"; +import type { + SqlColumnQueryRelation, + SqlColumnQuerySiteResult, +} from "./column-query-site.js"; +import type { + SqlColumnCatalogProviderReport, + SqlColumnCatalogFailure, + SqlCompletionIssue, + SqlCompletionItem, + SqlCompletionList, +} from "./relation-completion-types.js"; +import type { SqlRelationDialectRuntime } from "./relation-dialect.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, +} from "./types.js"; + +type ReadyColumnSite = Extract< + SqlColumnQuerySiteResult, + { readonly status: "ready" } +>; + +export interface SqlPreparedColumnCatalogRelations { + readonly coverage: "complete" | "partial"; + readonly references: readonly SqlColumnCatalogRelationReference[]; + readonly relationsByRequestKey: ReadonlyMap< + string, + SqlColumnQueryRelation + >; +} + +export interface SqlColumnCompletionComposition { + readonly sources: readonly SqlColumnCatalogProviderReport[]; + readonly value: SqlCompletionList; +} + +function sameIdentifier( + dialect: SqlRelationDialectRuntime, + left: SqlIdentifierComponent, + right: SqlIdentifierComponent, +): boolean { + return dialect.completion.compareCteIdentifiers(left, right) === "equal"; +} + +function pathEndsWith( + dialect: SqlRelationDialectRuntime, + path: SqlIdentifierPath, + suffix: SqlIdentifierPath, +): boolean { + if (suffix.length === 0 || suffix.length > path.length) return false; + const offset = path.length - suffix.length; + return suffix.every((component, index) => { + const candidate = path[offset + index]; + return candidate !== undefined && + sameIdentifier(dialect, candidate, component); + }); +} + +function relationMatchesQualifier( + dialect: SqlRelationDialectRuntime, + relation: SqlColumnQueryRelation, + qualifier: SqlIdentifierPath, +): boolean { + if (qualifier.length === 0) return true; + if ( + qualifier.length === 1 && + relation.alias !== null && + qualifier[0] !== undefined && + sameIdentifier(dialect, relation.alias, qualifier[0]) + ) { + return true; + } + return relation.alias === null && + pathEndsWith(dialect, relation.path, qualifier); +} + +export function prepareSqlColumnCatalogRelations( + site: ReadyColumnSite, + dialect: SqlRelationDialectRuntime, +): SqlPreparedColumnCatalogRelations { + const references: SqlColumnCatalogRelationReference[] = []; + const relationsByRequestKey = new Map< + string, + SqlColumnQueryRelation + >(); + let coverage: "complete" | "partial" = "complete"; + for (let index = 0; index < site.relations.length; index += 1) { + const relation = site.relations[index]; + if ( + relation === undefined || + !relationMatchesQualifier( + dialect, + relation, + site.qualifier, + ) + ) { + continue; + } + if (references.length === MAX_COLUMN_BATCH_RELATIONS) { + coverage = "partial"; + break; + } + const requestKey = `binding:${index}`; + references.push(Object.freeze({ + path: relation.path, + requestKey, + })); + relationsByRequestKey.set(requestKey, relation); + } + return Object.freeze({ + coverage, + references: Object.freeze(references), + relationsByRequestKey, + }); +} + +function relationLabel(relation: SqlColumnQueryRelation): string { + return relation.alias?.value ?? + relation.path.map((component) => component.value).join("."); +} + +function issue( + reason: Exclude, +): SqlCompletionIssue { + return Object.freeze({ reason }); +} + +function resultIssues( + relation: SqlColumnCatalogRelationResult, +): readonly SqlCompletionIssue[] { + if (relation.status === "loading") { + return Object.freeze([issue("column-catalog-loading")]); + } + if (relation.status === "failed") { + return Object.freeze([issue("column-catalog-failed")]); + } + return relation.coverage === "partial" + ? Object.freeze([issue("column-catalog-partial")]) + : Object.freeze([]); +} + +function list( + items: readonly SqlCompletionItem[], + issues: readonly SqlCompletionIssue[], +): SqlCompletionList { + const first = issues[0]; + if (first === undefined) { + return Object.freeze({ + isIncomplete: false, + issues: Object.freeze([] as const), + items: Object.freeze(items), + }); + } + const incompleteIssues: [ + SqlCompletionIssue, + ...SqlCompletionIssue[], + ] = [first, ...issues.slice(1)]; + return Object.freeze({ + isIncomplete: true, + issues: Object.freeze(incompleteIssues), + items: Object.freeze(items), + }); +} + +function unavailableComposition( + providerId: string, + reason: Extract< + SqlColumnCatalogBatchOutcome, + { readonly status: "unavailable" } + >["reason"], +): SqlColumnCompletionComposition { + return Object.freeze({ + sources: Object.freeze([Object.freeze({ + feature: "column-catalog", + outcome: "unavailable", + providerId, + reason, + })]), + value: list([], [issue( + reason === "malformed-response" + ? "column-catalog-malformed" + : "column-catalog-failed", + )]), + }); +} + +function compareItems( + left: SqlCompletionItem, + right: SqlCompletionItem, +): number { + return compareText(left.label, right.label) || + compareText(left.detail ?? "", right.detail ?? "") || + compareText(left.edit.insert, right.edit.insert) || + left.edit.from - right.edit.from || + left.edit.to - right.edit.to; +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +export function composeSqlColumnCompletion( + input: { + readonly dialect: SqlRelationDialectRuntime; + readonly outcome: SqlColumnCatalogBatchOutcome; + readonly prepared: SqlPreparedColumnCatalogRelations; + readonly providerId: string; + readonly site: ReadyColumnSite; + }, +): SqlColumnCompletionComposition | null { + if ( + input.outcome.status === "cancelled" || + input.outcome.status === "superseded" + ) { + return null; + } + if (input.outcome.status === "unavailable") { + return unavailableComposition( + input.providerId, + input.outcome.reason, + ); + } + const items: SqlCompletionItem[] = []; + const issues: SqlCompletionIssue[] = []; + let hasPartial = + input.site.coverage === "partial" || + input.prepared.coverage === "partial"; + if (hasPartial) issues.push(issue("query-binding-partial")); + let hasLoading = false; + let hasFailure = false; + const seen = new Set(); + const failures: SqlColumnCatalogFailure[] = []; + for (const result of input.outcome.relations) { + issues.push(...resultIssues(result)); + if (result.status === "loading") { + hasLoading = true; + continue; + } + if (result.status === "failed") { + hasFailure = true; + failures.push(Object.freeze({ + code: result.code, + requestKey: result.requestKey, + retry: result.retry, + })); + continue; + } + if (result.coverage === "partial") hasPartial = true; + const relation = input.prepared.relationsByRequestKey.get( + result.requestKey, + ); + if (!relation) { + issues.push(issue("column-catalog-malformed")); + continue; + } + for (const column of result.columns) { + if ( + input.dialect.completion.cteIdentifierMatchesPrefix( + column.identifier, + input.site.prefix, + ) !== "match" + ) { + continue; + } + const identity = [ + column.provenance.providerId, + column.provenance.scope, + column.provenance.relationEntityId, + column.provenance.columnEntityId, + column.insertText, + ].join("\u0000"); + if (seen.has(identity)) continue; + seen.add(identity); + const relationName = relationLabel(relation); + const metadata = column.detail ?? column.dataType; + items.push(Object.freeze({ + ...(column.dataType === undefined + ? {} + : { dataType: column.dataType }), + detail: metadata === undefined + ? relationName + : `${metadata} — ${relationName}`, + edit: Object.freeze({ + from: input.site.replacementRange.from, + insert: column.insertText, + to: input.site.replacementRange.to, + }), + kind: "column", + label: column.identifier.value, + provenance: Object.freeze({ + columnEntityId: column.provenance.columnEntityId, + epoch: column.provenance.epoch, + kind: "column-catalog", + providerId: column.provenance.providerId, + relationEntityId: column.provenance.relationEntityId, + scope: column.provenance.scope, + }), + relationRequestKey: result.requestKey, + })); + } + } + const deduplicatedIssues = Array.from( + new Map(issues.map((item) => [item.reason, item])).values(), + ); + const coverage = hasPartial || hasFailure ? "partial" : "complete"; + const frozenFailures = Object.freeze(failures); + const firstFailure = failures[0]; + const source: SqlColumnCatalogProviderReport = hasLoading + ? { + feature: "column-catalog", + failures: frozenFailures, + outcome: "loading", + providerId: input.outcome.providerId, + } + : hasFailure && items.length === 0 && + firstFailure !== undefined + ? { + feature: "column-catalog", + failures: Object.freeze([ + firstFailure, + ...failures.slice(1), + ]), + outcome: "failed", + providerId: input.outcome.providerId, + } + : { + coverage, + feature: "column-catalog", + failures: frozenFailures, + outcome: "ready", + providerId: input.outcome.providerId, + }; + return Object.freeze({ + sources: Object.freeze([Object.freeze(source)]), + value: list( + Object.freeze(items.sort(compareItems)), + Object.freeze(deduplicatedIssues), + ), + }); +} diff --git a/src/vnext/column-query-site.ts b/src/vnext/column-query-site.ts new file mode 100644 index 0000000..dc8e8a6 --- /dev/null +++ b/src/vnext/column-query-site.ts @@ -0,0 +1,681 @@ +import { + BoundedSqlLexer, + type BoundedSqlLexeme, +} from "./bounded-sql-lexer.js"; +import type { SqlRelationDialectRuntime } from "./relation-dialect.js"; +import { isSqlRelationDialectRuntime } from "./relation-runtime-auth.js"; +import { + isSqlSourceSnapshot, + type SqlSourceSnapshot, +} from "./source.js"; +import { + isSqlStatementSlotSnapshot, + type SqlStatementSlot, +} from "./statement-index.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, + SqlTextRange, +} from "./types.js"; + +export const MAX_COLUMN_QUERY_RELATIONS = 256; + +export interface SqlColumnQueryRelation { + readonly alias: SqlIdentifierComponent | null; + readonly path: SqlIdentifierPath; + readonly range: SqlTextRange; +} + +export type SqlColumnQuerySiteIssue = + | "derived-relation" + | "incomplete-relation" + | "nested-query" + | "opaque-template-context" + | "table-function"; + +export type SqlColumnQuerySiteResult = + | { + readonly status: "inactive"; + readonly reason: + | "cursor-in-comment" + | "cursor-in-string" + | "cursor-in-embedded-region" + | "not-column-position" + | "not-select-query"; + } + | { + readonly status: "unavailable"; + readonly reason: + | "ambiguous-query-site" + | "opaque-statement" + | "resource-limit"; + } + | { + readonly status: "ready"; + readonly coverage: "complete" | "partial"; + readonly issues: readonly SqlColumnQuerySiteIssue[]; + readonly prefix: SqlIdentifierComponent; + readonly qualifier: SqlIdentifierPath; + readonly relations: readonly SqlColumnQueryRelation[]; + readonly replacementRange: SqlTextRange; + }; + +interface Token extends BoundedSqlLexeme { + readonly depth: number; +} + +type Clause = + | "from" + | "group" + | "having" + | "join-condition" + | "limit" + | "order" + | "qualify" + | "select-list" + | "where"; + +const RELATION_END_WORDS: ReadonlySet = new Set([ + "cross", + "except", + "fetch", + "full", + "group", + "having", + "inner", + "intersect", + "join", + "left", + "limit", + "natural", + "offset", + "on", + "order", + "qualify", + "right", + "union", + "using", + "where", + "window", +]); + +function inactive( + reason: Extract< + SqlColumnQuerySiteResult, + { status: "inactive" } + >["reason"], +): SqlColumnQuerySiteResult { + return Object.freeze({ reason, status: "inactive" }); +} + +function unavailable( + reason: Extract< + SqlColumnQuerySiteResult, + { status: "unavailable" } + >["reason"], +): SqlColumnQuerySiteResult { + return Object.freeze({ reason, status: "unavailable" }); +} + +function word( + source: SqlSourceSnapshot, + token: Token | undefined, +): string | null { + return token?.kind === "word" + ? source.analysisText.slice(token.from, token.to).toLowerCase() + : null; +} + +function isIdentifier(token: Token | undefined): boolean { + return token?.kind === "word" || + token?.kind === "quoted-identifier"; +} + +function punctuation( + source: SqlSourceSnapshot, + token: Token | undefined, + expected: string, +): boolean { + return token?.kind === "punctuation" && + source.analysisText.slice(token.from, token.to) === expected; +} + +function tokenize( + source: SqlSourceSnapshot, + slot: Extract, + dialect: SqlRelationDialectRuntime, +): readonly Token[] | null { + const lexer = new BoundedSqlLexer( + source, + slot.source.from, + slot.source.to, + dialect.querySite.lexicalProfile, + ); + const tokens: Token[] = []; + let depth = 0; + for (let lexeme = lexer.next(); lexeme; lexeme = lexer.next()) { + if (punctuation(source, { ...lexeme, depth }, ")")) { + depth = Math.max(0, depth - 1); + } + tokens.push(Object.freeze({ ...lexeme, depth })); + if (punctuation(source, { ...lexeme, depth }, "(")) { + depth += 1; + } + } + return lexer.resource === null ? Object.freeze(tokens) : null; +} + +function decodePath( + source: SqlSourceSnapshot, + dialect: SqlRelationDialectRuntime, + from: number, + to: number, +): SqlIdentifierPath | null { + const raw = source.analysisText.slice(from, to); + const decoded = dialect.querySite.decodeRelationPath(raw, raw.length); + if ( + decoded.status !== "decoded" || + decoded.prefix.value.length === 0 + ) { + return null; + } + return Object.freeze([ + ...decoded.qualifier, + decoded.prefix, + ]); +} + +function parseNamedRelation( + source: SqlSourceSnapshot, + dialect: SqlRelationDialectRuntime, + tokens: readonly Token[], + start: number, + depth: number, +): { + readonly next: number; + readonly relation: SqlColumnQueryRelation | null; + readonly issue: SqlColumnQuerySiteIssue | null; +} { + const first = tokens[start]; + if (!isIdentifier(first) || first?.depth !== depth) { + return { + issue: punctuation(source, first, "(") + ? "derived-relation" + : "incomplete-relation", + next: start + 1, + relation: null, + }; + } + let end = start + 1; + while ( + punctuation(source, tokens[end], ".") && + tokens[end]?.depth === depth && + isIdentifier(tokens[end + 1]) && + tokens[end + 1]?.depth === depth + ) { + end += 2; + } + const lastPathToken = tokens[end - 1] ?? first; + const path = decodePath( + source, + dialect, + first.from, + lastPathToken.to, + ); + if (path === null) { + return { + issue: "incomplete-relation", + next: end, + relation: null, + }; + } + if (punctuation(source, tokens[end], "(")) { + return { + issue: "table-function", + next: end + 1, + relation: null, + }; + } + let alias: SqlIdentifierComponent | null = null; + const maybeAs = word(source, tokens[end]); + if (maybeAs === "as") end += 1; + const aliasToken = tokens[end]; + const aliasWord = aliasToken ? word(source, aliasToken) : null; + if ( + isIdentifier(aliasToken) && + aliasToken?.depth === depth && + (aliasWord === null || + !RELATION_END_WORDS.has(aliasWord)) + ) { + const aliasPath = decodePath( + source, + dialect, + aliasToken.from, + aliasToken.to, + ); + if (aliasPath?.length === 1) { + alias = aliasPath[0] ?? null; + end += 1; + } + } + return { + issue: + maybeAs === "as" && alias === null + ? "incomplete-relation" + : null, + next: end, + relation: Object.freeze({ + alias, + path, + range: Object.freeze({ + from: first.from, + to: lastPathToken.to, + }), + }), + }; +} + +function clauseAt( + source: SqlSourceSnapshot, + tokens: readonly Token[], + selectIndex: number, + depth: number, + position: number, +): Clause { + let clause: Clause = "select-list"; + for ( + let index = selectIndex + 1; + index < tokens.length && tokens[index]!.from < position; + index += 1 + ) { + const token = tokens[index]!; + if (token.depth !== depth) continue; + switch (word(source, token)) { + case "from": + clause = "from"; + break; + case "group": + clause = "group"; + break; + case "having": + clause = "having"; + break; + case "limit": + case "fetch": + case "offset": + clause = "limit"; + break; + case "on": + case "using": + clause = "join-condition"; + break; + case "order": + clause = "order"; + break; + case "qualify": + clause = "qualify"; + break; + case "where": + clause = "where"; + break; + } + } + return clause; +} + +function typedPath( + source: SqlSourceSnapshot, + tokens: readonly Token[], + position: number, + dialect: SqlRelationDialectRuntime, +): { + readonly prefix: SqlIdentifierComponent; + readonly qualifier: SqlIdentifierPath; + readonly replacementRange: SqlTextRange; +} | null { + let endIndex = tokens.findIndex((token) => + token.from < position && position <= token.to + ); + const endToken = tokens[endIndex]; + if ( + endIndex < 0 || + (!isIdentifier(endToken) && + !punctuation(source, endToken, ".")) + ) { + return Object.freeze({ + prefix: Object.freeze({ quoted: false, value: "" }), + qualifier: Object.freeze([]), + replacementRange: Object.freeze({ from: position, to: position }), + }); + } + let startIndex = endIndex; + let expectIdentifier = punctuation(source, endToken, "."); + while (startIndex > 0) { + const previous = tokens[startIndex - 1]; + if ( + expectIdentifier + ? !isIdentifier(previous) + : !punctuation(source, previous, ".") + ) { + break; + } + startIndex -= 1; + expectIdentifier = !expectIdentifier; + } + const from = tokens[startIndex]?.from ?? position; + const raw = source.analysisText.slice(from, position); + const decoded = dialect.querySite.decodeRelationPath(raw, raw.length); + if (decoded.status !== "decoded") return null; + return Object.freeze({ + prefix: decoded.prefix, + qualifier: decoded.qualifier, + replacementRange: Object.freeze({ + from: from + decoded.finalSegment.from, + to: from + decoded.finalSegment.to, + }), + }); +} + +function cursorTokenReason( + tokens: readonly Token[], + position: number, +): Extract< + SqlColumnQuerySiteResult, + { status: "inactive" } +>["reason"] | null { + const token = tokens.find((candidate) => + candidate.from <= position && + (position < candidate.to || + (candidate.kind === "line-comment" && + position === candidate.to)) + ); + switch (token?.kind) { + case "barrier": + return "cursor-in-embedded-region"; + case "comment": + case "line-comment": + return "cursor-in-comment"; + case "string": + return "cursor-in-string"; + default: + return null; + } +} + +function collectRelations( + source: SqlSourceSnapshot, + dialect: SqlRelationDialectRuntime, + tokens: readonly Token[], + selectIndex: number, + visibilityPosition: number, + precedingOnly: boolean, + relations: SqlColumnQueryRelation[], + issues: Set, +): boolean { + const selectDepth = tokens[selectIndex]?.depth; + if (selectDepth === undefined) return false; + const clause = clauseAt( + source, + tokens, + selectIndex, + selectDepth, + visibilityPosition, + ); + let commaStartsRelation = false; + let inFrom = false; + for ( + let index = selectIndex + 1; + index < tokens.length; + index += 1 + ) { + const token = tokens[index]!; + if ( + (precedingOnly || clause === "join-condition") && + token.from >= visibilityPosition + ) { + break; + } + if (token.depth < selectDepth) break; + if (token.depth !== selectDepth) { + if (word(source, token) === "select") issues.add("nested-query"); + continue; + } + const tokenWord = word(source, token); + if ( + tokenWord === "union" || + tokenWord === "intersect" || + tokenWord === "except" + ) { + break; + } + if ( + tokenWord === "where" || + tokenWord === "group" || + tokenWord === "having" || + tokenWord === "qualify" || + tokenWord === "order" || + tokenWord === "limit" + ) { + inFrom = false; + commaStartsRelation = false; + continue; + } + if (tokenWord === "on" || tokenWord === "using") { + commaStartsRelation = false; + continue; + } + const startsRelation = + tokenWord === "from" || + tokenWord === "join" || + (inFrom && + commaStartsRelation && + punctuation(source, token, ",")); + if (!startsRelation) continue; + inFrom = true; + let relationIndex = index + 1; + if ( + (dialect.id === "postgresql" || dialect.id === "duckdb") && + word(source, tokens[relationIndex]) === "lateral" + ) { + relationIndex += 1; + } + const parsed = parseNamedRelation( + source, + dialect, + tokens, + relationIndex, + selectDepth, + ); + if (parsed.issue !== null) issues.add(parsed.issue); + if (parsed.relation !== null) { + relations.push(parsed.relation); + if (relations.length > MAX_COLUMN_QUERY_RELATIONS) return false; + } + commaStartsRelation = true; + index = Math.max(index, parsed.next - 1); + } + return true; +} + +function openingBefore( + source: SqlSourceSnapshot, + tokens: readonly Token[], + beforeIndex: number, + depth: number, +): number { + for (let index = beforeIndex - 1; index >= 0; index -= 1) { + const token = tokens[index]!; + if ( + token.depth === depth && + punctuation(source, token, "(") + ) { + return index; + } + } + return -1; +} + +function correlatedParentSelectIndex( + source: SqlSourceSnapshot, + dialect: SqlRelationDialectRuntime, + tokens: readonly Token[], + childIndex: number, +): readonly [selectIndex: number, precedingOnly: boolean] | null { + const child = tokens[childIndex]; + if (!child || child.depth === 0) return null; + for (let depth = child.depth - 1; depth >= 0; depth -= 1) { + const lowerBound = depth === 0 + ? -1 + : tokens[ + openingBefore(source, tokens, childIndex, depth - 1) + ]?.from ?? -1; + for (let index = childIndex - 1; index >= 0; index -= 1) { + const token = tokens[index]!; + if (token.from <= lowerBound) break; + if (token.depth === depth && word(source, token) === "select") { + const openingIndex = openingBefore( + source, + tokens, + childIndex, + depth, + ); + const opening = tokens[openingIndex]?.from ?? -1; + if (opening <= token.from) return null; + const inFrom = + clauseAt( + source, + tokens, + index, + depth, + opening, + ) === "from"; + if (!inFrom) return [index, false]; + return (dialect.id === "postgresql" || dialect.id === "duckdb") && + word(source, tokens[openingIndex - 1]) === "lateral" + ? [index, true] + : null; + } + } + } + return null; +} + +export function recognizeSqlColumnQuerySite( + source: SqlSourceSnapshot, + slot: SqlStatementSlot, + position: number, + dialect: SqlRelationDialectRuntime, +): SqlColumnQuerySiteResult { + if ( + !isSqlSourceSnapshot(source) || + !isSqlStatementSlotSnapshot(slot) || + !isSqlRelationDialectRuntime(dialect) || + !Number.isSafeInteger(position) || + position < 0 || + position > source.analysisText.length + ) { + return unavailable("ambiguous-query-site"); + } + if (slot.boundaryQuality === "opaque") { + return unavailable("opaque-statement"); + } + if ( + position < slot.source.from || + position > slot.source.to + ) { + return inactive("not-column-position"); + } + const tokens = tokenize(source, slot, dialect); + if (tokens === null) return unavailable("resource-limit"); + const cursorReason = cursorTokenReason(tokens, position); + if (cursorReason !== null) return inactive(cursorReason); + + let cursorDepth = 0; + for (const token of tokens) { + if (token.from >= position) break; + cursorDepth = token.depth; + if (punctuation(source, token, "(")) cursorDepth += 1; + } + let selectIndex = -1; + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]!; + if ( + token.from <= position && + token.depth <= cursorDepth && + word(source, token) === "select" + ) { + selectIndex = index; + } + } + if (selectIndex < 0) return inactive("not-select-query"); + const selectDepth = tokens[selectIndex]!.depth; + const clause = clauseAt( + source, + tokens, + selectIndex, + selectDepth, + position, + ); + if (clause === "from" || clause === "limit") { + return inactive("not-column-position"); + } + const path = typedPath(source, tokens, position, dialect); + if (path === null) { + return inactive("not-column-position"); + } + + const relations: SqlColumnQueryRelation[] = []; + const issues = new Set(); + if ( + !collectRelations( + source, + dialect, + tokens, + selectIndex, + position, + false, + relations, + issues, + ) + ) { + return unavailable("resource-limit"); + } + let childIndex = selectIndex; + for (;;) { + const parent = correlatedParentSelectIndex( + source, + dialect, + tokens, + childIndex, + ); + if (parent === null) break; + const [parentIndex, precedingOnly] = parent; + const childPosition = tokens[childIndex]?.from; + if ( + childPosition === undefined || + !collectRelations( + source, + dialect, + tokens, + parentIndex, + childPosition, + precedingOnly, + relations, + issues, + ) + ) { + return unavailable("resource-limit"); + } + childIndex = parentIndex; + } + const issueList = Object.freeze(Array.from(issues).sort()); + return Object.freeze({ + coverage: issueList.length === 0 ? "complete" : "partial", + issues: issueList, + prefix: path.prefix, + qualifier: path.qualifier, + relations: Object.freeze(relations), + replacementRange: path.replacementRange, + status: "ready", + }); +} diff --git a/src/vnext/index.ts b/src/vnext/index.ts index 5a98211..d5bc733 100644 --- a/src/vnext/index.ts +++ b/src/vnext/index.ts @@ -5,6 +5,32 @@ export { duckdbDialect, postgresDialect, } from "./session.js"; +export type { + SqlColumnCatalogBatchRequest, + SqlColumnCatalogBatchResponse, + SqlColumnCatalogColumn, + SqlColumnCatalogCoverage, + SqlColumnCatalogProvider, + SqlColumnCatalogProvenance, + SqlColumnCatalogProviderRelationResult, + SqlColumnCatalogProviderResponse, + SqlColumnCatalogRelationReference, + SqlColumnCatalogRelationResult, + SqlColumnCatalogResolvedColumn, +} from "./column-catalog-types.js"; +export type { + SqlCanonicalNamespacePath, + SqlNamespaceCatalogContainer, + SqlNamespaceCatalogProvider, + SqlNamespaceCatalogProviderResponse, + SqlNamespaceCatalogProvenance, + SqlNamespaceCatalogResolvedContainer, + SqlNamespaceCatalogSearchRequest, + SqlNamespaceCatalogSearchResponse, + SqlNamespaceContainerRole, + SqlNamespacePathComponent, + SqlNamespaceQuerySite, +} from "./namespace-catalog-types.js"; export type { OpenSqlDocument, SqlCatalogContext, @@ -53,9 +79,15 @@ export type { SqlCatalogSearchRequest, SqlCatalogSearchResponse, SqlCompletionCancellationReason, + SqlColumnCatalogProviderReport, + SqlColumnCatalogFailure, + SqlColumnCompletionProvenance, + SqlCompletionProviderReport, SqlCompletionIssue, SqlCompletionRequest, SqlCompletionRefreshToken, + SqlNamespaceCatalogProviderReport, + SqlNamespaceCompletionProvenance, SqlCompletionTrigger, SqlDisposable, SqlRelationCatalogProvider, diff --git a/src/vnext/local-relation-site.ts b/src/vnext/local-relation-site.ts index 5199eb0..27ad6cd 100644 --- a/src/vnext/local-relation-site.ts +++ b/src/vnext/local-relation-site.ts @@ -4,6 +4,10 @@ import { type SqlCteVisibility, visibleSqlCtesAt, } from "./cte-layout.js"; +import { + recognizeSqlColumnQuerySite, + type SqlColumnQuerySiteResult, +} from "./column-query-site.js"; import { recognizeSqlRelationQuerySiteWithEntrypoints, type SqlQuerySiteResult, @@ -179,13 +183,6 @@ export function analyzeSqlLocalRelationSite( }); } const relativePosition = position - context.slot.source.from; - if ( - !Number.isSafeInteger(relativePosition) || - relativePosition < 0 || - relativePosition > context.layout.statementLength - ) { - return unavailableSite(); - } return Object.freeze({ local: Object.freeze({ cteVisibility: visibleSqlCtesAt( @@ -198,3 +195,54 @@ export function analyzeSqlLocalRelationSite( status: "ready", }); } + +export function analyzeSqlLocalColumnSite( + statement: SqlLocalRelationStatement, + position: number, +): SqlColumnQuerySiteResult { + if ( + statement === null || + typeof statement !== "object" + ) { + return Object.freeze({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + } + const context = localRelationStatements.get(statement); + if (!context) { + return Object.freeze({ + reason: "ambiguous-query-site", + status: "unavailable", + }); + } + const result = recognizeSqlColumnQuerySite( + context.source, + context.slot, + position, + context.dialect, + ); + if (result.status !== "ready") return result; + const visibility = visibleSqlCtesAt( + context.layout, + position - context.slot.source.from, + ); + const relations = result.relations.filter((relation) => { + const name = relation.path.length === 1 + ? relation.path[0] + : undefined; + return name === undefined || + !visibility.ctes.some((cte) => + context.dialect.completion.compareCteIdentifiers( + name, + cte.name, + ) === "equal" + ); + }); + return relations.length === result.relations.length + ? result + : Object.freeze({ + ...result, + relations: Object.freeze(relations), + }); +} diff --git a/src/vnext/namespace-catalog-boundary.ts b/src/vnext/namespace-catalog-boundary.ts new file mode 100644 index 0000000..efe718b --- /dev/null +++ b/src/vnext/namespace-catalog-boundary.ts @@ -0,0 +1,575 @@ +import type { SqlCatalogEpoch } from "./relation-completion-types.js"; +import type { + SqlCanonicalNamespacePath, + SqlNamespaceCatalogContainer, + SqlNamespaceCatalogResolvedContainer, + SqlNamespaceCatalogSearchRequest, + SqlNamespaceCatalogSearchResponse, + SqlNamespaceContainerRole, + SqlNamespacePathComponent, +} from "./namespace-catalog-types.js"; +import { + isDataArray, + type SqlIdentifierComponent, + type SqlIdentifierPath, +} from "./types.js"; + +export const MAX_NAMESPACE_PROVIDER_ID_LENGTH = 256; +export const MAX_NAMESPACE_SCOPE_LENGTH = 512; +export const MAX_NAMESPACE_DIALECT_ID_LENGTH = 128; +export const MAX_NAMESPACE_EPOCH_TOKEN_LENGTH = 256; +export const MAX_NAMESPACE_ENTITY_ID_LENGTH = 256; +export const MAX_NAMESPACE_IDENTIFIER_LENGTH = 256; +export const MAX_NAMESPACE_INSERT_TEXT_LENGTH = 1_024; +export const MAX_NAMESPACE_DETAIL_LENGTH = 1_024; +export const MAX_NAMESPACE_PATH_COMPONENTS = 32; +export const MAX_NAMESPACE_SEARCH_PATHS = 32; +export const MAX_NAMESPACE_SEARCH_PATH_COMPONENTS = 8; +export const MAX_NAMESPACE_RESULTS = 128; + +const capturedProviderBrand: unique symbol = Symbol( + "CapturedSqlNamespaceCatalogProvider", +); + +export interface CapturedSqlNamespaceCatalogProvider { + readonly [capturedProviderBrand]: + "CapturedSqlNamespaceCatalogProvider"; +} + +export interface SqlCapturedNamespaceCatalogProviderContext { + readonly id: string; + readonly search: ( + this: void, + request: SqlNamespaceCatalogSearchRequest, + signal: AbortSignal, + ) => unknown; +} + +export type SqlNamespaceBoundaryFailureReason = + | "duplicate-entity-id" + | "invalid-shape" + | "resource-limit"; + +export type SqlNamespaceBoundaryResult = + | { + readonly status: "accepted"; + readonly value: Value; + } + | { + readonly reason: SqlNamespaceBoundaryFailureReason; + readonly status: "malformed"; + }; + +interface DataRecord { + readonly fields: ReadonlyMap; +} + +const capturedProviders = new WeakMap< + object, + { readonly id: string; readonly search: Function } +>(); + +function accepted( + value: Value, +): SqlNamespaceBoundaryResult { + return Object.freeze({ status: "accepted", value }); +} + +function malformed( + reason: SqlNamespaceBoundaryFailureReason, +): SqlNamespaceBoundaryResult { + return Object.freeze({ reason, status: "malformed" }); +} + +function record( + value: unknown, + allowed: ReadonlySet, +): DataRecord | null { + if ( + value === null || + typeof value !== "object" + ) return null; + let keys: readonly PropertyKey[]; + try { + keys = Reflect.ownKeys(value); + } catch { + return null; + } + const fields = new Map(); + for (const key of keys) { + if (typeof key !== "string" || !allowed.has(key)) return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, key); + } catch { + return null; + } + if (!descriptor || !("value" in descriptor)) return null; + fields.set(key, descriptor.value); + } + return { fields }; +} + +function required( + source: DataRecord, + key: string, +): unknown { + return source.fields.has(key) + ? source.fields.get(key) + : undefined; +} + +function boundedString( + value: unknown, + maximum: number, + allowEmpty = false, +): string | null { + return typeof value === "string" && + value.length <= maximum && + (allowEmpty || value.length > 0) + ? value + : null; +} + +function arrayLength( + value: unknown, + maximum: number, +): number | null { + if (!isDataArray(value)) return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, "length"); + } catch { + return null; + } + const length = descriptor && "value" in descriptor + ? descriptor.value + : undefined; + return typeof length === "number" && + Number.isSafeInteger(length) && + length >= 0 && + length <= maximum + ? length + : null; +} + +function arrayElement( + value: unknown, + index: number, +): unknown | null { + if (!isDataArray(value)) return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, String(index)); + } catch { + return null; + } + return descriptor && "value" in descriptor + ? descriptor.value + : null; +} + +function identifier( + value: unknown, + allowEmpty = false, +): SqlIdentifierComponent | null { + const source = record(value, new Set(["quoted", "value"])); + if (!source) return null; + const quoted = required(source, "quoted"); + const text = boundedString( + required(source, "value"), + MAX_NAMESPACE_IDENTIFIER_LENGTH, + allowEmpty, + ); + return typeof quoted === "boolean" && text !== null + ? Object.freeze({ quoted, value: text }) + : null; +} + +function identifierPath( + value: unknown, + maximum: number, +): SqlIdentifierPath | null { + const length = arrayLength(value, maximum); + if (length === null) return null; + const output: SqlIdentifierComponent[] = []; + for (let index = 0; index < length; index += 1) { + const component = identifier(arrayElement(value, index)); + if (!component) return null; + output.push(component); + } + return Object.freeze(output); +} + +function searchPaths( + value: unknown, +): readonly SqlIdentifierPath[] | null { + const length = arrayLength(value, MAX_NAMESPACE_SEARCH_PATHS); + if (length === null) return null; + const output: SqlIdentifierPath[] = []; + for (let index = 0; index < length; index += 1) { + const path = identifierPath( + arrayElement(value, index), + MAX_NAMESPACE_SEARCH_PATH_COMPONENTS, + ); + if (!path) return null; + output.push(path); + } + return Object.freeze(output); +} + +function epoch(value: unknown): SqlCatalogEpoch | null { + const source = record(value, new Set(["generation", "token"])); + if (!source) return null; + const generation = required(source, "generation"); + const token = boundedString( + required(source, "token"), + MAX_NAMESPACE_EPOCH_TOKEN_LENGTH, + ); + return typeof generation === "number" && + Number.isSafeInteger(generation) && + generation >= 0 && + token !== null + ? Object.freeze({ generation, token }) + : null; +} + +function sameEpoch( + left: SqlCatalogEpoch, + right: SqlCatalogEpoch, +): boolean { + return left.generation === right.generation && + left.token === right.token; +} + +function role(value: unknown): SqlNamespaceContainerRole | null { + return value === "catalog" || + value === "schema" || + value === "project" || + value === "dataset" + ? value + : null; +} + +function namespacePath( + value: unknown, +): SqlCanonicalNamespacePath | null { + const length = arrayLength(value, MAX_NAMESPACE_PATH_COMPONENTS); + if (length === null || length === 0) return null; + const output: SqlNamespacePathComponent[] = []; + for (let index = 0; index < length; index += 1) { + const source = record( + arrayElement(value, index), + new Set(["quoted", "role", "value"]), + ); + if (!source) return null; + const componentRole = role(required(source, "role")); + const component = identifier({ + quoted: required(source, "quoted"), + value: required(source, "value"), + }); + if (!componentRole || !component) return null; + output.push(Object.freeze({ ...component, role: componentRole })); + } + const first = output[0]; + if (!first) return null; + return Object.freeze([first, ...output.slice(1)]); +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function comparePath( + left: SqlCanonicalNamespacePath, + right: SqlCanonicalNamespacePath, +): number { + const length = Math.min(left.length, right.length); + for (let index = 0; index < length; index += 1) { + const leftComponent = left[index]; + const rightComponent = right[index]; + if (!leftComponent || !rightComponent) continue; + const compared = + compareText(leftComponent.role, rightComponent.role) || + Number(rightComponent.quoted) - Number(leftComponent.quoted) || + compareText(leftComponent.value, rightComponent.value); + if (compared !== 0) return compared; + } + return left.length - right.length; +} + +function sameContainer( + left: SqlNamespaceCatalogContainer, + right: SqlNamespaceCatalogContainer, +): boolean { + return left.containerEntityId === right.containerEntityId && + left.detail === right.detail && + left.insertText === right.insertText && + left.matchQuality === right.matchQuality && + comparePath(left.canonicalPath, right.canonicalPath) === 0; +} + +export function captureSqlNamespaceCatalogProvider( + value: unknown, +): SqlNamespaceBoundaryResult { + const source = record(value, new Set(["id", "search"])); + const id = source + ? boundedString( + required(source, "id"), + MAX_NAMESPACE_PROVIDER_ID_LENGTH, + ) + : null; + const search = source ? required(source, "search") : null; + if (!source || id === null || typeof search !== "function") { + return malformed("invalid-shape"); + } + const capturedValue: CapturedSqlNamespaceCatalogProvider = { + [capturedProviderBrand]: "CapturedSqlNamespaceCatalogProvider", + }; + const captured = Object.freeze(capturedValue); + capturedProviders.set(captured, { id, search }); + return accepted(captured); +} + +export function resolveSqlNamespaceCatalogProvider( + provider: CapturedSqlNamespaceCatalogProvider, +): SqlCapturedNamespaceCatalogProviderContext | null { + const captured = capturedProviders.get(provider); + return captured + ? Object.freeze({ + id: captured.id, + search: ( + request: SqlNamespaceCatalogSearchRequest, + signal: AbortSignal, + ): unknown => + Reflect.apply(captured.search, undefined, [request, signal]), + }) + : null; +} + +export function createSqlNamespaceCatalogSearchRequest( + value: unknown, +): SqlNamespaceBoundaryResult { + const source = record( + value, + new Set([ + "dialectId", + "expectedEpoch", + "limit", + "prefix", + "qualifier", + "scope", + "searchPaths", + ]), + ); + if (!source) return malformed("invalid-shape"); + const dialectId = boundedString( + required(source, "dialectId"), + MAX_NAMESPACE_DIALECT_ID_LENGTH, + ); + const scope = boundedString( + required(source, "scope"), + MAX_NAMESPACE_SCOPE_LENGTH, + ); + const expectedValue = required(source, "expectedEpoch"); + const expectedEpoch = expectedValue === null + ? null + : epoch(expectedValue); + const limit = required(source, "limit"); + const prefix = identifier(required(source, "prefix"), true); + const qualifier = identifierPath( + required(source, "qualifier"), + MAX_NAMESPACE_PATH_COMPONENTS, + ); + const paths = searchPaths(required(source, "searchPaths")); + if ( + dialectId === null || + scope === null || + expectedEpoch === null && expectedValue !== null || + typeof limit !== "number" || + !Number.isSafeInteger(limit) || + limit < 1 || + limit > MAX_NAMESPACE_RESULTS || + !prefix || + !qualifier || + !paths + ) { + return malformed("invalid-shape"); + } + return accepted(Object.freeze({ + dialectId, + expectedEpoch, + limit, + prefix, + qualifier, + scope, + searchPaths: paths, + })); +} + +function decodeContainer( + value: unknown, + providerId: string, + scope: string, + responseEpoch: SqlCatalogEpoch, +): SqlNamespaceCatalogResolvedContainer | null { + const source = record( + value, + new Set([ + "canonicalPath", + "containerEntityId", + "detail", + "insertText", + "matchQuality", + ]), + ); + if (!source) return null; + const canonicalPath = namespacePath( + required(source, "canonicalPath"), + ); + const containerEntityId = boundedString( + required(source, "containerEntityId"), + MAX_NAMESPACE_ENTITY_ID_LENGTH, + ); + const insertText = boundedString( + required(source, "insertText"), + MAX_NAMESPACE_INSERT_TEXT_LENGTH, + ); + const matchQuality = required(source, "matchQuality"); + const rawDetail = required(source, "detail"); + const detail = rawDetail === undefined + ? undefined + : boundedString(rawDetail, MAX_NAMESPACE_DETAIL_LENGTH, true); + if ( + !canonicalPath || + containerEntityId === null || + insertText === null || + (matchQuality !== "exact" && matchQuality !== "equivalent") || + detail === null + ) { + return null; + } + return Object.freeze({ + canonicalPath, + containerEntityId, + ...(detail === undefined ? {} : { detail }), + insertText, + matchQuality, + provenance: Object.freeze({ + containerEntityId, + epoch: responseEpoch, + providerId, + scope, + }), + }); +} + +export function decodeSqlNamespaceCatalogSearchResponse( + provider: CapturedSqlNamespaceCatalogProvider, + request: SqlNamespaceCatalogSearchRequest, + value: unknown, +): SqlNamespaceBoundaryResult { + const context = resolveSqlNamespaceCatalogProvider(provider); + if (!context) return malformed("invalid-shape"); + const source = record( + value, + new Set([ + "code", + "containers", + "coverage", + "epoch", + "retry", + "status", + ]), + ); + if (!source) return malformed("invalid-shape"); + const responseEpoch = epoch(required(source, "epoch")); + if ( + !responseEpoch || + request.expectedEpoch !== null && + !sameEpoch(request.expectedEpoch, responseEpoch) + ) { + return malformed("invalid-shape"); + } + const status = required(source, "status"); + if (status === "loading") { + if (source.fields.size !== 2) return malformed("invalid-shape"); + return accepted(Object.freeze({ + epoch: responseEpoch, + status, + })); + } + if (status === "failed") { + const code = required(source, "code"); + const retry = required(source, "retry"); + if ( + source.fields.size !== 4 || + ( + code !== "authentication" && + code !== "authorization" && + code !== "invalid-configuration" && + code !== "rate-limited" && + code !== "unavailable" && + code !== "unknown" + ) || + ( + retry !== "after-invalidation" && + retry !== "never" && + retry !== "next-request" + ) + ) { + return malformed("invalid-shape"); + } + return accepted(Object.freeze({ + code, + epoch: responseEpoch, + retry, + status, + })); + } + const coverage = required(source, "coverage"); + const rawContainers = required(source, "containers"); + const length = arrayLength(rawContainers, request.limit); + if ( + status !== "ready" || + source.fields.size !== 4 || + (coverage !== "complete" && coverage !== "partial") || + length === null + ) { + return malformed("invalid-shape"); + } + const byId = new Map< + string, + SqlNamespaceCatalogResolvedContainer + >(); + for (let index = 0; index < length; index += 1) { + const container = decodeContainer( + arrayElement(rawContainers, index), + context.id, + request.scope, + responseEpoch, + ); + if (!container) return malformed("invalid-shape"); + const prior = byId.get(container.containerEntityId); + if (prior && !sameContainer(prior, container)) { + return malformed("duplicate-entity-id"); + } + byId.set(container.containerEntityId, container); + } + const containers = [...byId.values()].sort((left, right) => + (left.matchQuality === right.matchQuality + ? 0 + : left.matchQuality === "exact" + ? -1 + : 1) || + left.canonicalPath.length - right.canonicalPath.length || + comparePath(left.canonicalPath, right.canonicalPath) || + compareText(left.containerEntityId, right.containerEntityId) + ); + return accepted(Object.freeze({ + containers: Object.freeze(containers), + coverage, + epoch: responseEpoch, + status, + })); +} diff --git a/src/vnext/namespace-catalog-coordinator.ts b/src/vnext/namespace-catalog-coordinator.ts new file mode 100644 index 0000000..ca9e731 --- /dev/null +++ b/src/vnext/namespace-catalog-coordinator.ts @@ -0,0 +1,501 @@ +import { + captureSqlNamespaceCatalogProvider, + createSqlNamespaceCatalogSearchRequest, + decodeSqlNamespaceCatalogSearchResponse, + MAX_NAMESPACE_DIALECT_ID_LENGTH, + MAX_NAMESPACE_SCOPE_LENGTH, + resolveSqlNamespaceCatalogProvider, + type CapturedSqlNamespaceCatalogProvider, + type SqlCapturedNamespaceCatalogProviderContext, +} from "./namespace-catalog-boundary.js"; +import type { + SqlNamespaceCatalogSearchRequest, + SqlNamespaceCatalogSearchResponse, +} from "./namespace-catalog-types.js"; +import type { SqlCatalogEpoch } from "./relation-completion-types.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, +} from "./types.js"; + +export const DEFAULT_NAMESPACE_CACHE_ENTRIES = 256; +export const MAX_NAMESPACE_CACHE_ENTRIES = 4_096; + +export interface SqlNamespaceCatalogOwnerOptions { + readonly dialectId: string; + readonly scope: string; +} + +export interface SqlNamespaceCatalogSearchInput { + readonly expectedEpoch: SqlCatalogEpoch | null; + readonly limit: number; + readonly prefix: SqlIdentifierComponent; + readonly qualifier: SqlIdentifierPath; + readonly searchPaths: readonly SqlIdentifierPath[]; +} + +export type SqlNamespaceCatalogSearchOutcome = + | { + readonly providerId: string; + readonly response: SqlNamespaceCatalogSearchResponse; + readonly scope: string; + readonly status: "usable"; + } + | { + readonly status: "cancelled"; + } + | { + readonly status: "superseded"; + } + | { + readonly reason: + | "disposed" + | "invalid-request" + | "malformed-response" + | "provider-failed"; + readonly status: "unavailable"; + }; + +export interface SqlNamespaceCatalogSearchTicket { + readonly cancel: (this: void) => void; + readonly result: Promise; +} + +export interface SqlNamespaceCatalogOwner { + readonly dispose: (this: void) => void; + readonly request: ( + this: void, + input: SqlNamespaceCatalogSearchInput, + ) => SqlNamespaceCatalogSearchTicket; +} + +export type SqlNamespaceCatalogOwnerResult = + | { + readonly owner: SqlNamespaceCatalogOwner; + readonly status: "prepared"; + } + | { + readonly reason: "disposed" | "invalid-options"; + readonly status: "unavailable"; + }; + +export interface SqlNamespaceCatalogCoordinator { + readonly dispose: (this: void) => void; + readonly prepareOwner: ( + this: void, + options: SqlNamespaceCatalogOwnerOptions, + ) => SqlNamespaceCatalogOwnerResult; + readonly providerId: string; +} + +export type SqlNamespaceCatalogCoordinatorResult = + | { + readonly coordinator: SqlNamespaceCatalogCoordinator; + readonly status: "created"; + } + | { + readonly reason: "invalid-options" | "invalid-provider"; + readonly status: "unavailable"; + }; + +interface CoordinatorState { + readonly map: Map; + off: boolean; + readonly id: string; + readonly limit: number; + readonly load: SqlCapturedNamespaceCatalogProviderContext["search"]; + readonly pool: Set; + readonly wire: CapturedSqlNamespaceCatalogProvider; +} + +interface OwnerState { + job: ConsumerState | null; + readonly lang: string; + gen: number; + off: boolean; + rev: SqlCatalogEpoch | null; + root: CoordinatorState | null; + readonly scope: string; +} + +interface ConsumerState { + readonly abort: AbortController; + done: boolean; + end: + | ((value: SqlNamespaceCatalogSearchOutcome) => void) + | null; + readonly host: OwnerState; +} + +const CANCELLED: SqlNamespaceCatalogSearchOutcome = + Object.freeze({ status: "cancelled" }); +const SUPERSEDED: SqlNamespaceCatalogSearchOutcome = + Object.freeze({ status: "superseded" }); + +function unavailable( + reason: Extract< + SqlNamespaceCatalogSearchOutcome, + { readonly status: "unavailable" } + >["reason"], +): SqlNamespaceCatalogSearchOutcome { + return Object.freeze({ reason, status: "unavailable" }); +} + +function property( + value: unknown, + key: string, +): { readonly value: unknown } | null { + if (value === null || typeof value !== "object") return null; + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(value, key); + } catch { + return null; + } + return descriptor && "value" in descriptor + ? { value: descriptor.value } + : null; +} + +function boundedString(value: unknown, maximum: number): string | null { + return typeof value === "string" && + value.length > 0 && + value.length <= maximum + ? value + : null; +} + +function segment(value: string): string { + return `${value.length}:${value}`; +} + +function componentKey(component: SqlIdentifierComponent): string { + return segment( + `${component.quoted ? "q" : "u"}${component.value}`, + ); +} + +function pathKey(path: SqlIdentifierPath): string { + return segment(path.map(componentKey).join("")); +} + +function cacheKey( + request: SqlNamespaceCatalogSearchRequest, + responseEpoch: SqlCatalogEpoch, +): string { + return [ + segment(request.scope), + segment(request.dialectId), + segment(String(responseEpoch.generation)), + segment(responseEpoch.token), + pathKey(request.qualifier), + componentKey(request.prefix), + segment(request.searchPaths.map(pathKey).join("")), + segment(String(request.limit)), + ].join(""); +} + +function cacheGet( + state: CoordinatorState, + key: string, +): SqlNamespaceCatalogSearchResponse | null { + const value = state.map.get(key); + if (!value) return null; + state.map.delete(key); + state.map.set(key, value); + return value; +} + +function cacheSet( + state: CoordinatorState, + key: string, + value: SqlNamespaceCatalogSearchResponse, +): void { + state.map.delete(key); + state.map.set(key, value); + while (state.map.size > state.limit) { + for (const oldest of state.map.keys()) { + state.map.delete(oldest); + break; + } + } +} + +function settle( + consumer: ConsumerState, + outcome: SqlNamespaceCatalogSearchOutcome, +): void { + if (consumer.host.job === consumer) { + consumer.host.job = null; + } + if (consumer.done) return; + consumer.done = true; + const resolve = consumer.end; + consumer.end = null; + resolve?.(outcome); +} + +function cancel( + consumer: ConsumerState, + outcome: SqlNamespaceCatalogSearchOutcome, +): void { + if (consumer.done) return; + consumer.abort.abort(); + settle(consumer, outcome); +} + +function settledTicket( + outcome: SqlNamespaceCatalogSearchOutcome, +): SqlNamespaceCatalogSearchTicket { + return Object.freeze({ + cancel: (): void => {}, + result: Promise.resolve(outcome), + }); +} + +function providerWork( + state: CoordinatorState, + owner: OwnerState, + request: SqlNamespaceCatalogSearchRequest, + consumer: ConsumerState, +): void { + let pending: unknown; + try { + pending = state.load( + request, + consumer.abort.signal, + ); + } catch { + settle(consumer, unavailable("provider-failed")); + return; + } + Promise.resolve(pending).then( + (value) => { + if ( + consumer.done || + consumer.abort.signal.aborted || + state.off || + owner.off || + owner.root !== state + ) { + return; + } + const decoded = decodeSqlNamespaceCatalogSearchResponse( + state.wire, + request, + value, + ); + if (decoded.status === "malformed") { + settle(consumer, unavailable("malformed-response")); + return; + } + owner.rev = decoded.value.epoch; + if ( + decoded.value.status === "ready" && + decoded.value.coverage === "complete" + ) { + cacheSet( + state, + cacheKey(request, decoded.value.epoch), + decoded.value, + ); + } + settle(consumer, Object.freeze({ + providerId: state.id, + response: decoded.value, + scope: owner.scope, + status: "usable", + })); + }, + () => { + if (!consumer.done) { + settle(consumer, unavailable("provider-failed")); + } + }, + ); +} + +function request( + owner: OwnerState, + input: unknown, +): SqlNamespaceCatalogSearchTicket { + const state = owner.root; + if (!state || state.off || owner.off) { + return settledTicket(unavailable("disposed")); + } + const expected = property(input, "expectedEpoch"); + const limit = property(input, "limit"); + const prefix = property(input, "prefix"); + const qualifier = property(input, "qualifier"); + const paths = property(input, "searchPaths"); + if (!expected || !limit || !prefix || !qualifier || !paths) { + return settledTicket(unavailable("invalid-request")); + } + const created = createSqlNamespaceCatalogSearchRequest({ + dialectId: owner.lang, + expectedEpoch: expected.value === null + ? owner.rev + : expected.value, + limit: limit.value, + prefix: prefix.value, + qualifier: qualifier.value, + scope: owner.scope, + searchPaths: paths.value, + }); + if (created.status === "malformed") { + return settledTicket(unavailable("invalid-request")); + } + owner.gen += 1; + const generation = owner.gen; + if (owner.job) cancel(owner.job, SUPERSEDED); + if (generation !== owner.gen) { + return settledTicket( + owner.off ? unavailable("disposed") : SUPERSEDED, + ); + } + const normalized = created.value; + const key = normalized.expectedEpoch === null + ? null + : cacheKey(normalized, normalized.expectedEpoch); + const cached = key === null ? null : cacheGet(state, key); + if (cached) { + return settledTicket(Object.freeze({ + providerId: state.id, + response: cached, + scope: owner.scope, + status: "usable", + })); + } + let resolveResult: ( + value: SqlNamespaceCatalogSearchOutcome + ) => void = (): void => {}; + const result = new Promise( + (resolve) => { + resolveResult = resolve; + }, + ); + const consumer: ConsumerState = { + abort: new AbortController(), + done: false, + end: resolveResult, + host: owner, + }; + owner.job = consumer; + const ticket = Object.freeze({ + cancel: (): void => cancel(consumer, CANCELLED), + result, + }); + providerWork(state, owner, normalized, consumer); + return ticket; +} + +function disposeOwner(owner: OwnerState): void { + if (owner.off) return; + owner.off = true; + owner.gen += 1; + if (owner.job) cancel(owner.job, unavailable("disposed")); + owner.root?.pool.delete(owner); + owner.root = null; +} + +function prepareOwner( + state: CoordinatorState, + value: unknown, +): SqlNamespaceCatalogOwnerResult { + if (state.off) { + return Object.freeze({ reason: "disposed", status: "unavailable" }); + } + const dialectId = boundedString( + property(value, "dialectId")?.value, + MAX_NAMESPACE_DIALECT_ID_LENGTH, + ); + const scope = boundedString( + property(value, "scope")?.value, + MAX_NAMESPACE_SCOPE_LENGTH, + ); + if (!dialectId || !scope) { + return Object.freeze({ + reason: "invalid-options", + status: "unavailable", + }); + } + const owner: OwnerState = { + job: null, + gen: 0, + lang: dialectId, + off: false, + rev: null, + root: state, + scope, + }; + state.pool.add(owner); + return Object.freeze({ + owner: Object.freeze({ + dispose: (): void => disposeOwner(owner), + request: (input: SqlNamespaceCatalogSearchInput) => + request(owner, input), + }), + status: "prepared", + }); +} + +function dispose(state: CoordinatorState): void { + if (state.off) return; + state.off = true; + state.map.clear(); + for (const owner of state.pool) disposeOwner(owner); +} + +export function createSqlNamespaceCatalogCoordinator( + value: unknown, +): SqlNamespaceCatalogCoordinatorResult { + const captured = captureSqlNamespaceCatalogProvider( + property(value, "provider")?.value, + ); + if (captured.status === "malformed") { + return Object.freeze({ + reason: "invalid-provider", + status: "unavailable", + }); + } + const context = resolveSqlNamespaceCatalogProvider(captured.value); + if (!context) { + return Object.freeze({ + reason: "invalid-provider", + status: "unavailable", + }); + } + const rawMaximum = property(value, "maxCacheEntries"); + const maximum = rawMaximum === null + ? DEFAULT_NAMESPACE_CACHE_ENTRIES + : rawMaximum.value; + if ( + typeof maximum !== "number" || + !Number.isSafeInteger(maximum) || + maximum < 1 || + maximum > MAX_NAMESPACE_CACHE_ENTRIES + ) { + return Object.freeze({ + reason: "invalid-options", + status: "unavailable", + }); + } + const state: CoordinatorState = { + id: context.id, + limit: maximum, + load: context.search, + map: new Map(), + off: false, + pool: new Set(), + wire: captured.value, + }; + return Object.freeze({ + coordinator: Object.freeze({ + dispose: (): void => dispose(state), + prepareOwner: (options: SqlNamespaceCatalogOwnerOptions) => + prepareOwner(state, options), + providerId: state.id, + }), + status: "created", + }); +} diff --git a/src/vnext/namespace-catalog-types.ts b/src/vnext/namespace-catalog-types.ts new file mode 100644 index 0000000..bcf62f1 --- /dev/null +++ b/src/vnext/namespace-catalog-types.ts @@ -0,0 +1,108 @@ +import type { SqlCatalogEpoch } from "./relation-completion-types.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, + SqlTextRange, +} from "./types.js"; + +export type SqlNamespaceContainerRole = + | "catalog" + | "schema" + | "project" + | "dataset"; + +export interface SqlNamespacePathComponent + extends SqlIdentifierComponent { + readonly role: SqlNamespaceContainerRole; +} + +export type SqlCanonicalNamespacePath = + readonly [ + SqlNamespacePathComponent, + ...SqlNamespacePathComponent[], + ]; + +export interface SqlNamespaceCatalogSearchRequest { + readonly dialectId: string; + readonly expectedEpoch: SqlCatalogEpoch | null; + readonly limit: number; + readonly prefix: SqlIdentifierComponent; + readonly qualifier: SqlIdentifierPath; + readonly scope: string; + readonly searchPaths: readonly SqlIdentifierPath[]; +} + +export interface SqlNamespaceCatalogContainer { + readonly canonicalPath: SqlCanonicalNamespacePath; + readonly containerEntityId: string; + readonly detail?: string; + readonly insertText: string; + readonly matchQuality: "equivalent" | "exact"; +} + +export type SqlNamespaceCatalogSearchResponse = + | { + readonly containers: + readonly SqlNamespaceCatalogResolvedContainer[]; + readonly coverage: "complete" | "partial"; + readonly epoch: SqlCatalogEpoch; + readonly status: "ready"; + } + | { + readonly epoch: SqlCatalogEpoch; + readonly status: "loading"; + } + | { + readonly code: + | "authentication" + | "authorization" + | "invalid-configuration" + | "rate-limited" + | "unavailable" + | "unknown"; + readonly epoch: SqlCatalogEpoch; + readonly retry: + | "after-invalidation" + | "never" + | "next-request"; + readonly status: "failed"; + }; + +export type SqlNamespaceCatalogProviderResponse = + | { + readonly containers: readonly SqlNamespaceCatalogContainer[]; + readonly coverage: "complete" | "partial"; + readonly epoch: SqlCatalogEpoch; + readonly status: "ready"; + } + | Extract< + SqlNamespaceCatalogSearchResponse, + { readonly status: "loading" | "failed" } + >; + +export interface SqlNamespaceCatalogProvenance { + readonly containerEntityId: string; + readonly epoch: SqlCatalogEpoch; + readonly providerId: string; + readonly scope: string; +} + +export interface SqlNamespaceCatalogResolvedContainer + extends SqlNamespaceCatalogContainer { + readonly provenance: SqlNamespaceCatalogProvenance; +} + +export interface SqlNamespaceCatalogProvider { + readonly id: string; + readonly search: ( + this: void, + request: SqlNamespaceCatalogSearchRequest, + signal: AbortSignal, + ) => Promise; +} + +export interface SqlNamespaceQuerySite { + readonly prefix: SqlIdentifierComponent; + readonly qualifier: SqlIdentifierPath; + readonly replacementRange: SqlTextRange; +} diff --git a/src/vnext/namespace-completion.ts b/src/vnext/namespace-completion.ts new file mode 100644 index 0000000..d77215e --- /dev/null +++ b/src/vnext/namespace-completion.ts @@ -0,0 +1,270 @@ +import type { + SqlNamespaceCatalogSearchInput, + SqlNamespaceCatalogSearchOutcome, +} from "./namespace-catalog-coordinator.js"; +import type { + SqlCanonicalNamespacePath, + SqlNamespaceCatalogResolvedContainer, + SqlNamespaceContainerRole, + SqlNamespacePathComponent, + SqlNamespaceQuerySite, +} from "./namespace-catalog-types.js"; +import type { + SqlCatalogEpoch, + SqlNamespaceCatalogProviderReport, + SqlNamespaceCompletionProvenance, +} from "./relation-completion-types.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, + SqlTextRange, +} from "./types.js"; + +export interface SqlNamespaceCompletionItem { + readonly detail?: string; + readonly edit: { + readonly from: number; + readonly insert: string; + readonly to: number; + }; + readonly label: string; + readonly role: SqlNamespaceContainerRole; + readonly provenance: SqlNamespaceCompletionProvenance; +} + +export type SqlNamespaceCompletionIssue = + | "namespace-catalog-failed" + | "namespace-catalog-loading" + | "namespace-catalog-malformed" + | "namespace-catalog-partial" + | "namespace-prefix-uncertain" + | "result-limit"; + +export interface SqlNamespaceCompletionList { + readonly isIncomplete: boolean; + readonly issues: readonly SqlNamespaceCompletionIssue[]; + readonly items: readonly SqlNamespaceCompletionItem[]; +} + +export interface SqlNamespaceCompletionComposition { + readonly source: SqlNamespaceCatalogProviderReport; + readonly value: SqlNamespaceCompletionList; +} + +export type SqlNamespacePrefixMatcher = ( + this: void, + candidate: SqlIdentifierComponent, + prefix: SqlIdentifierComponent, +) => "match" | "no-match" | "unknown"; + +const ROLE_ORDER: Readonly> = + Object.freeze({ + catalog: 0, + project: 1, + schema: 2, + dataset: 3, + }); + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function finalComponent( + path: SqlCanonicalNamespacePath, +): SqlNamespacePathComponent { + let current = path[0]; + for (const component of path.slice(1)) current = component; + return current; +} + +function pathText( + container: SqlNamespaceCatalogResolvedContainer, +): string { + return container.canonicalPath.map((component) => + `${component.role}:${component.quoted ? "q" : "u"}:${component.value}` + ).join("\u0000"); +} + +function compareContainers( + left: SqlNamespaceCatalogResolvedContainer, + right: SqlNamespaceCatalogResolvedContainer, +): number { + const leftLast = finalComponent(left.canonicalPath); + const rightLast = finalComponent(right.canonicalPath); + return ( + (left.matchQuality === right.matchQuality + ? 0 + : left.matchQuality === "exact" + ? -1 + : 1) || + left.canonicalPath.length - right.canonicalPath.length || + ( + ROLE_ORDER[leftLast.role] - ROLE_ORDER[rightLast.role] + ) || + compareText(leftLast.value, rightLast.value) || + compareText(pathText(left), pathText(right)) || + compareText( + left.provenance.containerEntityId, + right.provenance.containerEntityId, + ) + ); +} + +function list( + items: readonly SqlNamespaceCompletionItem[], + issues: readonly SqlNamespaceCompletionIssue[], +): SqlNamespaceCompletionList { + return Object.freeze({ + isIncomplete: issues.length > 0, + issues: Object.freeze(issues), + items: Object.freeze(items), + }); +} + +export function prepareSqlNamespaceCatalogSearch( + site: SqlNamespaceQuerySite, + expectedEpoch: SqlCatalogEpoch | null, + searchPaths: readonly SqlIdentifierPath[], + limit: number, +): SqlNamespaceCatalogSearchInput { + return Object.freeze({ + expectedEpoch, + limit, + prefix: site.prefix, + qualifier: site.qualifier, + searchPaths, + }); +} + +function unavailable( + providerId: string, + reason: Extract< + SqlNamespaceCatalogSearchOutcome, + { readonly status: "unavailable" } + >["reason"], +): SqlNamespaceCompletionComposition { + return Object.freeze({ + source: Object.freeze({ + feature: "namespace-catalog", + outcome: "unavailable", + providerId, + reason, + }), + value: list( + [], + [reason === "malformed-response" + ? "namespace-catalog-malformed" + : "namespace-catalog-failed"], + ), + }); +} + +function item( + container: SqlNamespaceCatalogResolvedContainer, + replacementRange: SqlTextRange, +): SqlNamespaceCompletionItem { + const component = finalComponent(container.canonicalPath); + return Object.freeze({ + ...(container.detail === undefined + ? {} + : { detail: container.detail }), + edit: Object.freeze({ + from: replacementRange.from, + insert: container.insertText, + to: replacementRange.to, + }), + label: component.value, + provenance: Object.freeze({ + containerEntityId: container.provenance.containerEntityId, + epoch: container.provenance.epoch, + kind: "namespace-catalog", + providerId: container.provenance.providerId, + scope: container.provenance.scope, + }), + role: component.role, + }); +} + +export function composeSqlNamespaceCompletion( + input: { + readonly matchPrefix: SqlNamespacePrefixMatcher; + readonly outcome: SqlNamespaceCatalogSearchOutcome; + readonly prefix: SqlIdentifierComponent; + readonly providerId: string; + readonly replacementRange: SqlTextRange; + }, +): SqlNamespaceCompletionComposition | null { + if ( + input.outcome.status === "cancelled" || + input.outcome.status === "superseded" + ) { + return null; + } + if (input.outcome.status === "unavailable") { + return unavailable(input.providerId, input.outcome.reason); + } + const response = input.outcome.response; + if (response.status === "loading") { + return Object.freeze({ + source: Object.freeze({ + feature: "namespace-catalog", + outcome: "loading", + providerId: input.outcome.providerId, + }), + value: list([], ["namespace-catalog-loading"]), + }); + } + if (response.status === "failed") { + return Object.freeze({ + source: Object.freeze({ + code: response.code, + feature: "namespace-catalog", + outcome: "failed", + providerId: input.outcome.providerId, + retry: response.retry, + }), + value: list([], ["namespace-catalog-failed"]), + }); + } + const issues = new Set(); + if (response.coverage === "partial") { + issues.add("namespace-catalog-partial"); + } + const seen = new Set(); + const containers: SqlNamespaceCatalogResolvedContainer[] = []; + for (const container of response.containers) { + const component = finalComponent(container.canonicalPath); + let match: ReturnType; + try { + match = input.matchPrefix(component, input.prefix); + } catch { + match = "unknown"; + } + if (match === "unknown") { + issues.add("namespace-prefix-uncertain"); + continue; + } + if (match === "no-match") continue; + const identity = [ + container.provenance.providerId, + container.provenance.scope, + container.provenance.containerEntityId, + ].join("\u0000"); + if (seen.has(identity)) continue; + seen.add(identity); + containers.push(container); + } + containers.sort(compareContainers); + const items = containers.map((container) => + item(container, input.replacementRange) + ); + return Object.freeze({ + source: Object.freeze({ + coverage: response.coverage, + feature: "namespace-catalog", + outcome: "ready", + providerId: input.outcome.providerId, + }), + value: list(items, [...issues]), + }); +} diff --git a/src/vnext/node-sql-parser-adapter.ts b/src/vnext/node-sql-parser-adapter.ts index 86d80d6..197aa4f 100644 --- a/src/vnext/node-sql-parser-adapter.ts +++ b/src/vnext/node-sql-parser-adapter.ts @@ -5,6 +5,11 @@ import { type NodeSqlParserModuleLoadOutcome, type NodeSqlParserModuleLoader, } from "./node-sql-parser-backend.js"; +import { + normalizeNodeSqlParserQueryBindings, + type NodeSqlParserQueryBindingGrammar, +} from "./node-sql-parser-query-bindings.js"; +import type { SqlQueryBindingModel } from "./query-binding-model.js"; import { createCompatibilityParsedAnalysis, createFailedParserAnalysis, @@ -39,6 +44,7 @@ type NodeSqlParserPolicy = interface NodeSqlParserRuntime { readonly authority: SqlParserAuthority; readonly backend: NodeSqlParserBackend; + readonly grammar: NodeSqlParserQueryBindingGrammar; readonly limitations: readonly [ SqlCompatibilityLimitation, ...SqlCompatibilityLimitation[], @@ -61,7 +67,8 @@ type OwnDataProperty = class NodeSqlParserGlobalCleanupError extends Error {} class NodeSqlParserExecutionRealmError extends Error {} -const backendPayloads = new WeakMap(); +const queryBindingModels = + new WeakMap(); function readOwnDataProperty( value: object, @@ -307,7 +314,18 @@ function materializeBackendOutcome( statementText, runtime.authority, ); - backendPayloads.set(artifact, outcome.root); + const bindings = normalizeNodeSqlParserQueryBindings( + outcome.root, + statementText, + runtime.authority, + { + compatibility: runtime.policy === "dialect-compatibility", + grammar: runtime.grammar, + }, + ); + if (bindings.status === "ready") { + queryBindingModels.set(artifact, bindings.model); + } return createCompatibilityParsedAnalysis( artifact, runtime.limitations, @@ -349,6 +367,7 @@ function createAdapter(runtime: NodeSqlParserRuntime): SqlStatementParser { function createRuntime( backendIdentity: SqlSyntaxBackendIdentity, backend: NodeSqlParserBackend, + grammar: NodeSqlParserQueryBindingGrammar, policy: NodeSqlParserPolicy, ): NodeSqlParserRuntime { const authority = createSqlParserAuthority( @@ -359,6 +378,7 @@ function createRuntime( return Object.freeze({ authority, backend, + grammar, limitations: policy === "dialect-compatibility" ? DIALECT_COMPATIBILITY_LIMITATIONS @@ -378,6 +398,7 @@ const postgresqlParser = createAdapter( createRuntime( postgresqlBackendIdentity, postgresqlBackend, + "postgresql", "target-grammar", ), ); @@ -385,6 +406,7 @@ const bigQueryParser = createAdapter( createRuntime( bigQueryBackendIdentity, bigQueryBackend, + "bigquery", "target-grammar", ), ); @@ -392,6 +414,7 @@ const duckDbParser = createAdapter( createRuntime( postgresqlBackendIdentity, postgresqlBackend, + "postgresql", "dialect-compatibility", ), ); @@ -449,12 +472,19 @@ export const exportedForTesting = Object.freeze({ createNodeSqlParserBackend( adaptModuleLoaderForTesting(loadModule), ), + "postgresql", policy, ), ); }, - hasBackendPayload(artifact: SqlSyntaxArtifact): boolean { - return backendPayloads.has(artifact); + hasBackendPayload(_artifact: SqlSyntaxArtifact): boolean { + return false; }, createSynchronousModuleLoader, }); + +export function getNodeSqlParserQueryBindingModel( + artifact: SqlSyntaxArtifact, +): SqlQueryBindingModel | null { + return queryBindingModels.get(artifact) ?? null; +} diff --git a/src/vnext/node-sql-parser-browser-executor.ts b/src/vnext/node-sql-parser-browser-executor.ts index f250ec6..0c02e35 100644 --- a/src/vnext/node-sql-parser-browser-executor.ts +++ b/src/vnext/node-sql-parser-browser-executor.ts @@ -7,6 +7,7 @@ import { type NodeSqlParserWireFailureCode, type NodeSqlParserWireGrammar, } from "./node-sql-parser-wire.js"; +import type { SqlQueryBindingModelData } from "./query-binding-model.js"; import type { SqlStatementKind } from "./syntax.js"; export interface NodeSqlParserBrowserExecutorLimits { @@ -68,6 +69,7 @@ export type NodeSqlParserBrowserExecutorFailureCode = export type NodeSqlParserBrowserExecutorOutcome = | { readonly kind: "parsed"; + readonly queryBindings: SqlQueryBindingModelData | null; readonly statementKind: SqlStatementKind; } | { @@ -836,6 +838,7 @@ export function createNodeSqlParserBrowserExecutor( request, Object.freeze({ kind: "parsed", + queryBindings: message.queryBindings, statementKind: message.statementKind, }), ); diff --git a/src/vnext/node-sql-parser-browser-worker-endpoint.ts b/src/vnext/node-sql-parser-browser-worker-endpoint.ts index 7023fbe..8af377f 100644 --- a/src/vnext/node-sql-parser-browser-worker-endpoint.ts +++ b/src/vnext/node-sql-parser-browser-worker-endpoint.ts @@ -3,6 +3,10 @@ import { type NodeSqlParserBackendOutcome, type NodeSqlParserModuleLoadOutcome, } from "./node-sql-parser-backend.js"; +import { + normalizeNodeSqlParserQueryBindings, +} from "./node-sql-parser-query-bindings.js"; +import type { SqlQueryBindingModel } from "./query-binding-model.js"; import { decodeNodeSqlParserWireRequest, encodeNodeSqlParserWireBackendOutcome, @@ -250,6 +254,10 @@ export function installNodeSqlParserBrowserWorkerEndpoint( createModuleLoader(loaders.postgresql), ), } satisfies Record); + const bindingAuthorities = Object.freeze({ + bigquery: Object.freeze({}), + postgresql: Object.freeze({}), + } satisfies Record); function closeEndpoint(): void { state = "closed"; @@ -293,11 +301,27 @@ export function installNodeSqlParserBrowserWorkerEndpoint( guard, ); state = closeAfterSettlement ? "closed" : "idle"; + let queryBindings: SqlQueryBindingModel | null = null; + if (outcome.kind === "parsed") { + const normalized = normalizeNodeSqlParserQueryBindings( + outcome.root, + request.text, + bindingAuthorities[request.grammar], + { + compatibility: false, + grammar: request.grammar, + }, + ); + if (normalized.status === "ready") { + queryBindings = normalized.model; + } + } try { scope.postMessage( encodeNodeSqlParserWireBackendOutcome( request.requestId, outcome, + queryBindings, ), ); } catch { diff --git a/src/vnext/node-sql-parser-query-bindings.ts b/src/vnext/node-sql-parser-query-bindings.ts new file mode 100644 index 0000000..786849e --- /dev/null +++ b/src/vnext/node-sql-parser-query-bindings.ts @@ -0,0 +1,1251 @@ +import { + BoundedSqlLexer, + type BoundedSqlLexeme, +} from "./bounded-sql-lexer.js"; +import { + BIGQUERY_SQL_LEXICAL_PROFILE, + POSTGRESQL_SQL_LEXICAL_PROFILE, +} from "./lexical.js"; +import { + createSqlQueryBindingModel, + MAX_QUERY_BINDING_BLOCKS, + MAX_QUERY_BINDINGS, + type SqlQueryBindingIssue, + type SqlQueryBindingModel, + type SqlRelationBinding, + type SqlRelationScope, + type SqlVisibilityRegion, +} from "./query-binding-model.js"; +import { createIdentitySqlSource } from "./source.js"; +import type { + SqlIdentifierComponent, + SqlIdentifierPath, + SqlTextRange, +} from "./types.js"; + +export type NodeSqlParserQueryBindingGrammar = + | "bigquery" + | "postgresql"; + +export interface NodeSqlParserQueryBindingOptions { + readonly compatibility: boolean; + readonly grammar: NodeSqlParserQueryBindingGrammar; +} + +export type NodeSqlParserQueryBindingResult = + | { + readonly model: SqlQueryBindingModel; + readonly status: "ready"; + } + | { + readonly reason: + | "malformed-ast" + | "not-query" + | "resource-limit" + | "unsupported-shape"; + readonly status: "unavailable"; + }; + +interface Token extends BoundedSqlLexeme { + readonly depth: number; + readonly text: string; +} + +interface SelectSkeleton { + readonly depth: number; + readonly from: number; + readonly selectToken: number; + readonly to: number; +} + +interface AstSelect { + readonly cteChildren: AstSelect[]; + readonly derivedChildren: AstSelect[]; + readonly kind: "compound" | "select"; + readonly node: object; + skeleton: SelectSkeleton | null; +} + +interface RelationSite { + readonly aliasToken: Token | null; + readonly derived: boolean; + readonly explicitAlias: boolean; + readonly from: number; + readonly lexicalIdentifiers: readonly SqlIdentifierComponent[]; + readonly to: number; +} + +interface BlockBuild { + readonly ast: AstSelect; + readonly id: number; + readonly parent: number | null; + readonly range: SqlTextRange; + readonly skeleton: SelectSkeleton; +} + +interface MutableCoverage { + queryBlocks: "complete" | "partial"; + relationBindings: "complete" | "partial"; + visibility: "complete" | "partial"; +} + +function phaseValue( + values: readonly Value[], + index: number, +): Value { + const value = values[index]; + if (value === undefined) { + throw new Error("Query binding normalization phase invariant failed"); + } + return value; +} + +function assignedSkeleton(ast: AstSelect): SelectSkeleton { + if (ast.skeleton === null) { + throw new Error("Query binding skeleton assignment invariant failed"); + } + return ast.skeleton; +} + +type OwnProperty = + | { readonly kind: "invalid" | "missing" } + | { readonly kind: "value"; readonly value: unknown }; + +const MAX_AST_ARRAY_LENGTH = 1_024; +const MAX_AST_PROPERTY_TEXT = 2_048; + +function ownProperty(value: object, key: PropertyKey): OwnProperty { + try { + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (descriptor === undefined) { + return { kind: "missing" }; + } + if (!("value" in descriptor)) { + return { kind: "invalid" }; + } + return { kind: "value", value: descriptor.value }; + } catch { + return { kind: "invalid" }; + } +} + +function objectProperty(value: object, key: PropertyKey): object | null { + const property = ownProperty(value, key); + return property.kind === "value" && + property.value !== null && + typeof property.value === "object" + ? property.value + : null; +} + +function stringProperty(value: object, key: PropertyKey): string | null { + const property = ownProperty(value, key); + return property.kind === "value" && + typeof property.value === "string" && + property.value.length > 0 && + property.value.length <= MAX_AST_PROPERTY_TEXT + ? property.value + : null; +} + +function arrayProperty( + value: object, + key: PropertyKey, +): readonly unknown[] | null { + const property = ownProperty(value, key); + if (property.kind === "missing") { + return []; + } + if (property.kind !== "value") { + return null; + } + if (property.value === null) { + return []; + } + try { + if (!Array.isArray(property.value)) { + return null; + } + } catch { + return null; + } + const length = ownProperty(property.value, "length"); + if ( + length.kind !== "value" || + typeof length.value !== "number" || + !Number.isSafeInteger(length.value) || + length.value < 0 || + length.value > MAX_AST_ARRAY_LENGTH + ) { + return null; + } + const items: unknown[] = []; + for (let index = 0; index < length.value; index += 1) { + const item = ownProperty(property.value, index); + if (item.kind !== "value") { + return null; + } + items.push(item.value); + } + return items; +} + +function word(token: Token | undefined, expected: string): boolean { + return ( + token?.kind === "word" && + token.text.toLowerCase() === expected + ); +} + +function tokenize( + text: string, + grammar: NodeSqlParserQueryBindingGrammar, +): readonly Token[] | null { + const source = createIdentitySqlSource(text); + const lexer = new BoundedSqlLexer( + source, + 0, + text.length, + grammar === "bigquery" + ? BIGQUERY_SQL_LEXICAL_PROFILE + : POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + const tokens: Token[] = []; + let depth = 0; + for (;;) { + const lexeme = lexer.next(); + if (lexeme === null) { + return lexer.resource === null ? tokens : null; + } + if ( + lexeme.kind === "comment" || + lexeme.kind === "line-comment" + ) { + continue; + } + const raw = text.slice(lexeme.from, lexeme.to); + if (raw === ")") { + depth -= 1; + if (depth < 0) { + return null; + } + } + tokens.push(Object.freeze({ + ...lexeme, + depth, + text: raw, + })); + if (raw === "(") { + depth += 1; + if (depth > MAX_QUERY_BINDING_BLOCKS) { + return null; + } + } + } +} + +function selectSkeletons( + tokens: readonly Token[], + statementLength: number, +): readonly SelectSkeleton[] { + const skeletons: SelectSkeleton[] = []; + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]; + if (!token || !word(token, "select")) { + continue; + } + let to = statementLength; + for (let cursor = index + 1; cursor < tokens.length; cursor += 1) { + const candidate = tokens[cursor]; + if ( + candidate?.text === ")" && + candidate.depth < token.depth + ) { + to = candidate.from; + break; + } + } + let from = 0; + if (token.depth > 0) { + for (let cursor = index - 1; cursor >= 0; cursor -= 1) { + const candidate = tokens[cursor]; + if ( + candidate?.text !== "(" || + candidate.depth !== token.depth - 1 + ) { + continue; + } + const close = matchingClose(tokens, cursor, candidate.depth); + if (close !== null && phaseValue(tokens, close).from < token.from) { + continue; + } + from = candidate.to; + break; + } + } + skeletons.push(Object.freeze({ + depth: token.depth, + from, + selectToken: index, + to, + })); + } + return skeletons; +} + +function astSelectNode(value: unknown): object | null { + if (value === null || typeof value !== "object") { + return null; + } + const type = stringProperty(value, "type"); + return type !== null && + (type.toLowerCase() === "select" || + type.toLowerCase() === "union" || + type.toLowerCase() === "intersect" || + type.toLowerCase() === "except") + ? value + : null; +} + +function astSelectKind(value: object): "compound" | "select" { + return stringProperty(value, "type")?.toLowerCase() === "select" + ? "select" + : "compound"; +} + +function derivedAst(value: object): object | null { + const expression = objectProperty(value, "expr"); + if (expression === null) { + return null; + } + return astSelectNode(objectProperty(expression, "ast")); +} + +function buildAstTree( + root: object, +): { readonly root: AstSelect; readonly textual: readonly AstSelect[] } | null { + const seen = new WeakSet(); + const textual: AstSelect[] = []; + let count = 0; + + function visit(node: object): AstSelect | null { + if (seen.has(node) || count >= MAX_QUERY_BINDING_BLOCKS) { + return null; + } + seen.add(node); + count += 1; + const result: AstSelect = { + cteChildren: [], + derivedChildren: [], + kind: astSelectKind(node), + node, + skeleton: null, + }; + const withItems = arrayProperty(node, "with"); + if (withItems === null) { + return null; + } + for (const item of withItems) { + if (item === null || typeof item !== "object") { + return null; + } + const statement = astSelectNode(objectProperty(item, "stmt")); + if (statement === null) { + return null; + } + const child = visit(statement); + if (child === null) { + return null; + } + result.cteChildren.push(child); + } + textual.push(result); + const fromItems = arrayProperty(node, "from"); + if (fromItems === null) { + return null; + } + for (const item of fromItems) { + if (item === null || typeof item !== "object") { + return null; + } + const statement = derivedAst(item); + if (statement !== null) { + const child = visit(statement); + if (child === null) { + return null; + } + result.derivedChildren.push(child); + } + } + return result; + } + + const tree = visit(root); + return tree === null ? null : { root: tree, textual }; +} + +function assignSkeletons( + tree: { readonly root: AstSelect; readonly textual: readonly AstSelect[] }, + skeletons: readonly SelectSkeleton[], +): boolean { + if (tree.textual.length > skeletons.length) { + return false; + } + for (let index = 0; index < tree.textual.length; index += 1) { + const ast = phaseValue(tree.textual, index); + const skeleton = phaseValue(skeletons, index); + ast.skeleton = skeleton; + } + return true; +} + +function emitBlocks( + root: AstSelect, + statementLength: number, +): { + readonly blocks: readonly BlockBuild[]; + readonly ids: ReadonlyMap; +} | null { + const blocks: BlockBuild[] = []; + const ids = new Map(); + function emit(ast: AstSelect, parent: number | null): void { + const skeleton = assignedSkeleton(ast); + const id = blocks.length; + ids.set(ast.node, id); + blocks.push({ + ast, + id, + parent, + range: Object.freeze( + parent === null + ? { from: 0, to: statementLength } + : { from: skeleton.from, to: skeleton.to }, + ), + skeleton, + }); + for (const child of [...ast.cteChildren, ...ast.derivedChildren]) { + emit(child, id); + } + } + try { + emit(root, null); + return { blocks, ids }; + } catch { + return null; + } +} + +function tokenIdentifier(token: Token): SqlIdentifierComponent | null { + if (token.kind === "word") { + return Object.freeze({ quoted: false, value: token.text }); + } + if (token.kind !== "quoted-identifier" || token.text.length < 2) { + return null; + } + const quote = token.text[0]; + const value = token.text + .slice(1, -1) + .split(`${quote}${quote}`) + .join(quote); + return value.length === 0 + ? null + : Object.freeze({ quoted: true, value }); +} + +function matchingClose( + tokens: readonly Token[], + openIndex: number, + openDepth: number, +): number | null { + for (let index = openIndex + 1; index < tokens.length; index += 1) { + const token = tokens[index]; + if (token?.text === ")" && token.depth === openDepth) { + return index; + } + } + return null; +} + +function relationSites( + tokens: readonly Token[], + skeleton: SelectSkeleton, +): readonly RelationSite[] { + const sites: RelationSite[] = []; + let inFrom = false; + for ( + let index = skeleton.selectToken + 1; + index < tokens.length; + index += 1 + ) { + const token = tokens[index]; + if (!token || token.from >= skeleton.to) { + break; + } + if (token.depth !== skeleton.depth) { + continue; + } + const lower = token.kind === "word" + ? token.text.toLowerCase() + : ""; + if ( + lower === "where" || + lower === "group" || + lower === "having" || + lower === "qualify" || + lower === "order" || + lower === "limit" || + lower === "window" || + lower === "union" || + lower === "intersect" || + lower === "except" + ) { + inFrom = false; + } + const startsRelation = + lower === "from" || + lower === "join" || + (inFrom && token.text === ","); + if (!startsRelation) { + continue; + } + inFrom = true; + const startIndex = index + 1; + const start = tokens[startIndex]; + if (!start) { + continue; + } + let endIndex = startIndex; + let derived = false; + if (start.text === "(") { + const close = matchingClose(tokens, startIndex, start.depth); + if (close === null) { + continue; + } + endIndex = close; + derived = true; + } else { + while (endIndex + 2 < tokens.length) { + const dot = tokens[endIndex + 1]; + const component = tokens[endIndex + 2]; + if ( + dot?.text !== "." || + component === undefined || + tokenIdentifier(component) === null + ) { + break; + } + endIndex += 2; + } + } + const lexicalIdentifiers: SqlIdentifierComponent[] = []; + if (!derived) { + for ( + let componentIndex = startIndex; + componentIndex <= endIndex; + componentIndex += 1 + ) { + const componentToken = phaseValue(tokens, componentIndex); + const identifier = tokenIdentifier(componentToken); + if (identifier !== null) { + lexicalIdentifiers.push(identifier); + } + } + } + let aliasToken: Token | null = null; + let cursor = endIndex + 1; + const explicitAlias = word(tokens[cursor], "as"); + if (explicitAlias) { + cursor += 1; + } + const possibleAlias = tokens[cursor]; + if ( + possibleAlias !== undefined && + possibleAlias.depth === skeleton.depth && + tokenIdentifier(possibleAlias) !== null && + ![ + "cross", "full", "inner", "join", "left", "natural", + "on", "right", "using", "where", "group", "having", + "qualify", "order", "limit", "window", + ].includes(possibleAlias.text.toLowerCase()) + ) { + aliasToken = possibleAlias; + endIndex = cursor; + } + const end = phaseValue(tokens, endIndex); + sites.push(Object.freeze({ + aliasToken, + derived, + explicitAlias, + from: start.from, + lexicalIdentifiers: Object.freeze(lexicalIdentifiers), + to: end.to, + })); + } + return sites; +} + +function pathFromAst( + relation: object, + grammar: NodeSqlParserQueryBindingGrammar, + site: RelationSite, +): SqlIdentifierPath | null { + const table = stringProperty(relation, "table"); + if (table === null) { + return null; + } + const values: string[] = []; + const database = + stringProperty(relation, "db") ?? + stringProperty(relation, "schema"); + if (database !== null) { + values.push(database); + } + if (grammar === "bigquery") { + values.push(...table.split(".")); + } else { + values.push(table); + } + if ( + values.length === 0 || + values.length > 8 || + values.some((value) => value.length === 0 || value.length > 256) + ) { + return null; + } + const lexicalIdentifiers = site.lexicalIdentifiers; + const singleQuotedBigQueryPath = + grammar === "bigquery" && + lexicalIdentifiers.length === 1 && + lexicalIdentifiers[0]?.quoted === true; + if (singleQuotedBigQueryPath) { + if (lexicalIdentifiers[0]?.value !== values.join(".")) { + return null; + } + return Object.freeze(values.map((value) => + Object.freeze({ quoted: true, value }) + )); + } + if ( + values.length !== lexicalIdentifiers.length || + values.some((value, index) => { + const lexical = lexicalIdentifiers[index]; + return lexical === undefined || + !identifierMatchesAst(value, lexical); + }) + ) { + return null; + } + return Object.freeze([...lexicalIdentifiers]); +} + +interface CteDeclaration { + readonly name: SqlIdentifierComponent; + readonly range: SqlTextRange; +} + +interface CteClause { + readonly declarations: readonly CteDeclaration[]; + readonly recursion: "authenticated" | "none" | "uncertain"; +} + +interface CteEnvironment { + readonly uncertain: ReadonlySet; + readonly visible: ReadonlyMap; +} + +interface IdentifierToken { + readonly identifier: SqlIdentifierComponent; + readonly token: Token; +} + +function identifierMatchesAst( + astValue: string, + lexical: SqlIdentifierComponent, +): boolean { + return lexical.quoted + ? astValue === lexical.value + : astValue.toLowerCase() === lexical.value.toLowerCase(); +} + +function identifierKey(identifier: SqlIdentifierComponent): string { + return identifier.quoted + ? `quoted:${identifier.value}` + : `unquoted:${identifier.value.toLowerCase()}`; +} + +function findIdentifierToken( + tokens: readonly Token[], + from: number, + to: number, + depth: number, + astName: string, +): IdentifierToken | null { + for (const token of tokens) { + const identifier = tokenIdentifier(token); + if ( + token.from >= from && + token.from < to && + token.depth === depth && + identifier !== null && + identifierMatchesAst(astName, identifier) + ) { + return { identifier, token }; + } + } + return null; +} + +function astRecursiveMarker(item: object): boolean | null { + const property = ownProperty(item, "recursive"); + if (property.kind === "missing") { + return false; + } + return property.kind === "value" && + typeof property.value === "boolean" + ? property.value + : null; +} + +function ownCteDeclarations( + ast: AstSelect, + tokens: readonly Token[], +): CteClause | null { + const withItems = arrayProperty(ast.node, "with"); + if (withItems === null) { + return null; + } + if (withItems.length !== ast.cteChildren.length) { + return null; + } + const declarations: CteDeclaration[] = []; + let astRecursive = false; + let validMarkers = true; + let searchFrom = assignedSkeleton(ast).from; + for (let index = 0; index < withItems.length; index += 1) { + const item = withItems[index]; + const child = phaseValue(ast.cteChildren, index); + if (item === null || typeof item !== "object") { + return null; + } + const recursive = astRecursiveMarker(item); + if (recursive === null || (index > 0 && recursive)) { + validMarkers = false; + } else if (index === 0) { + astRecursive = recursive; + } + const nameObject = objectProperty(item, "name"); + const name = nameObject === null + ? stringProperty(item, "name") + : stringProperty(nameObject, "value"); + if (name === null) { + return null; + } + const declaration = findIdentifierToken( + tokens, + searchFrom, + assignedSkeleton(child).from, + assignedSkeleton(ast).depth, + name, + ); + if (declaration === null) { + return null; + } + declarations.push(Object.freeze({ + name: declaration.identifier, + range: Object.freeze({ + from: declaration.token.from, + to: declaration.token.to, + }), + })); + searchFrom = assignedSkeleton(child).to; + } + if (declarations.length === 0) { + return Object.freeze({ + declarations: Object.freeze(declarations), + recursion: "none", + }); + } + const first = phaseValue(declarations, 0); + const withIndex = tokens.findIndex((token) => + token.from >= assignedSkeleton(ast).from && + token.from < first.range.from && + token.depth === assignedSkeleton(ast).depth && + word(token, "with") + ); + if (withIndex < 0) { + return null; + } + const lexicalRecursive = word(tokens[withIndex + 1], "recursive"); + return Object.freeze({ + declarations: Object.freeze(declarations), + recursion: + validMarkers && lexicalRecursive === astRecursive + ? lexicalRecursive ? "authenticated" : "none" + : "uncertain", + }); +} + +function visibleCteDeclarations( + root: AstSelect, + tokens: readonly Token[], +): ReadonlyMap | null { + const result = new Map(); + function visit( + ast: AstSelect, + inherited: CteEnvironment, + ): boolean { + const clause = ownCteDeclarations(ast, tokens); + if (clause === null) { + return false; + } + const visible = new Map(inherited.visible); + const uncertain = new Set(inherited.uncertain); + for (let index = 0; index < ast.cteChildren.length; index += 1) { + const child = phaseValue(ast.cteChildren, index); + const declaration = phaseValue(clause.declarations, index); + const childVisible = new Map(visible); + const childUncertain = new Set(uncertain); + if (clause.recursion === "authenticated") { + childVisible.set(identifierKey(declaration.name), declaration); + for ( + let forward = index + 1; + forward < clause.declarations.length; + forward += 1 + ) { + const key = identifierKey( + phaseValue(clause.declarations, forward).name, + ); + childUncertain.add(key); + childVisible.delete(key); + } + } else if (clause.recursion === "uncertain") { + for (const candidate of clause.declarations) { + const key = identifierKey(candidate.name); + childUncertain.add(key); + childVisible.delete(key); + } + } + if (!visit(child, { + uncertain: childUncertain, + visible: childVisible, + })) { + return false; + } + visible.set(identifierKey(declaration.name), declaration); + uncertain.delete(identifierKey(declaration.name)); + } + const environment = { uncertain, visible }; + result.set(ast.node, environment); + for (const child of ast.derivedChildren) { + if (!visit(child, environment)) { + return false; + } + } + return true; + } + return visit(root, { + uncertain: new Set(), + visible: new Map(), + }) ? result : null; +} + +function addIssue( + issues: SqlQueryBindingIssue[], + code: SqlQueryBindingIssue["code"], + range: SqlTextRange, +): void { + if ( + issues.length < 256 && + !issues.some((issue) => + issue.code === code && + issue.range.from === range.from && + issue.range.to === range.to + ) + ) { + issues.push(Object.freeze({ code, range })); + } +} + +function clauseRegions( + tokens: readonly Token[], + block: BlockBuild, + fullScope: number, +): SqlVisibilityRegion[] { + const markers: { + readonly from: number; + readonly kind: SqlVisibilityRegion["kind"]; + readonly to: number; + }[] = []; + const start = block.skeleton.selectToken; + for (let index = start; index < tokens.length; index += 1) { + const token = tokens[index]; + if (!token || token.from >= block.skeleton.to) { + break; + } + if (token.depth !== block.skeleton.depth || token.kind !== "word") { + continue; + } + const value = token.text.toLowerCase(); + const kind = + value === "select" ? "select-list" + : value === "where" ? "where" + : value === "group" && word(tokens[index + 1], "by") ? "group-by" + : value === "having" ? "having" + : value === "qualify" ? "qualify" + : value === "order" && word(tokens[index + 1], "by") ? "order-by" + : value === "limit" ? "limit" + : null; + if (kind !== null) { + markers.push({ from: token.from, kind, to: token.to }); + } + } + const fromToken = tokens.find((token) => + token.depth === block.skeleton.depth && + token.from > block.skeleton.from && + token.from < block.skeleton.to && + word(token, "from") + ); + const result: SqlVisibilityRegion[] = []; + for (let index = 0; index < markers.length; index += 1) { + const marker = phaseValue(markers, index); + const next = markers[index + 1]; + const to = + marker.kind === "select-list" && fromToken !== undefined + ? fromToken.from + : next?.from ?? block.skeleton.to; + if (marker.to < to) { + result.push(Object.freeze({ + block: block.id, + kind: marker.kind, + range: Object.freeze({ from: marker.to, to }), + scope: fullScope, + })); + } + } + return result; +} + +function joinRegions( + tokens: readonly Token[], + block: BlockBuild, + sites: readonly RelationSite[], + scopesAfterRelations: readonly number[], +): SqlVisibilityRegion[] { + const result: SqlVisibilityRegion[] = []; + for (let relationIndex = 1; relationIndex < sites.length; relationIndex += 1) { + const site = phaseValue(sites, relationIndex); + const scope = scopesAfterRelations[relationIndex]; + if (scope === undefined) { + continue; + } + let conditionIndex = -1; + for (let index = 0; index < tokens.length; index += 1) { + const candidate = phaseValue(tokens, index); + if ( + candidate.from < site.to || + candidate.depth !== block.skeleton.depth + ) { + continue; + } + if (candidate.from >= block.skeleton.to) { + break; + } + if (word(candidate, "on") || word(candidate, "using")) { + conditionIndex = index; + break; + } + if ( + word(candidate, "join") || + candidate.text === "," || + word(candidate, "where") || + word(candidate, "group") || + word(candidate, "having") || + word(candidate, "qualify") || + word(candidate, "order") || + word(candidate, "limit") + ) { + break; + } + } + const token = tokens[conditionIndex]; + if (token === undefined) { + continue; + } + let to = block.skeleton.to; + for (let cursor = conditionIndex + 1; cursor < tokens.length; cursor += 1) { + const next = phaseValue(tokens, cursor); + if (next.from >= block.skeleton.to) { + break; + } + if ( + next.depth === block.skeleton.depth && + (word(next, "join") || + next.text === "," || + word(next, "where") || + word(next, "group") || + word(next, "having") || + word(next, "qualify") || + word(next, "order") || + word(next, "limit")) + ) { + to = next.from; + break; + } + } + if (token.to < to) { + result.push(Object.freeze({ + block: block.id, + kind: "join-condition", + range: Object.freeze({ from: token.to, to }), + scope, + })); + } + } + return result; +} + +export function normalizeNodeSqlParserQueryBindings( + root: unknown, + statementText: unknown, + authority: unknown, + options: NodeSqlParserQueryBindingOptions, +): NodeSqlParserQueryBindingResult { + if ( + typeof statementText !== "string" || + statementText.length === 0 || + statementText.length > 16 * 1024 || + authority === null || + typeof authority !== "object" + ) { + return Object.freeze({ reason: "resource-limit", status: "unavailable" }); + } + const astRoot = astSelectNode(root); + if (astRoot === null) { + return Object.freeze({ reason: "not-query", status: "unavailable" }); + } + const tokens = tokenize(statementText, options.grammar); + if (tokens === null) { + return Object.freeze({ reason: "resource-limit", status: "unavailable" }); + } + const skeletons = selectSkeletons(tokens, statementText.length); + const tree = buildAstTree(astRoot); + if (tree === null) { + return Object.freeze({ reason: "malformed-ast", status: "unavailable" }); + } + if (!assignSkeletons(tree, skeletons)) { + return Object.freeze({ reason: "unsupported-shape", status: "unavailable" }); + } + const emitted = emitBlocks(tree.root, statementText.length); + if (emitted === null) { + return Object.freeze({ reason: "unsupported-shape", status: "unavailable" }); + } + const declarations = visibleCteDeclarations(tree.root, tokens); + if (declarations === null) { + return Object.freeze({ reason: "malformed-ast", status: "unavailable" }); + } + + const coverage: MutableCoverage = { + queryBlocks: + skeletons.length === tree.textual.length ? "complete" : "partial", + relationBindings: "complete", + visibility: "complete", + }; + const issues: SqlQueryBindingIssue[] = []; + const statementRange = Object.freeze({ from: 0, to: statementText.length }); + if (options.compatibility) { + coverage.queryBlocks = "partial"; + coverage.relationBindings = "partial"; + coverage.visibility = "partial"; + addIssue(issues, "parser-compatibility", statementRange); + } + if (skeletons.length !== tree.textual.length) { + addIssue(issues, "unsupported-clause", statementRange); + } + + const bindings: SqlRelationBinding[] = []; + const scopes: SqlRelationScope[] = [ + Object.freeze({ addedBinding: null, parentScope: null }), + ]; + const regions: SqlVisibilityRegion[] = []; + const publicBlocks = emitted.blocks.map((block) => + Object.freeze({ + baseScope: 0, + kind: block.ast.kind, + parentBlock: block.parent, + range: block.range, + }), + ); + + for (const block of emitted.blocks) { + const fromItems = arrayProperty(block.ast.node, "from"); + if (fromItems === null) { + return Object.freeze({ reason: "malformed-ast", status: "unavailable" }); + } + const sites = relationSites(tokens, block.skeleton); + if (sites.length !== fromItems.length) { + coverage.relationBindings = "partial"; + coverage.visibility = "partial"; + addIssue(issues, "unsupported-relation-source", block.range); + } + let currentScope = 0; + const after: number[] = []; + const count = Math.min(sites.length, fromItems.length); + for (let index = 0; index < count; index += 1) { + if (bindings.length >= MAX_QUERY_BINDINGS) { + return Object.freeze({ reason: "resource-limit", status: "unavailable" }); + } + const rawRelation = fromItems[index]; + const site = phaseValue(sites, index); + if ( + rawRelation === null || + typeof rawRelation !== "object" + ) { + return Object.freeze({ reason: "malformed-ast", status: "unavailable" }); + } + const aliasText = stringProperty(rawRelation, "as"); + const aliasToken = site.aliasToken; + const aliasName = + aliasToken === null ? null : tokenIdentifier(aliasToken); + const aliasMatches = + aliasText !== null && + aliasName !== null && + identifierMatchesAst(aliasText, aliasName); + const alias = + aliasMatches && aliasName !== null && aliasToken !== null + ? Object.freeze({ + explicit: site.explicitAlias, + name: aliasName, + range: Object.freeze({ + from: aliasToken.from, + to: aliasToken.to, + }), + }) + : null; + if ( + (aliasText === null) !== (site.aliasToken === null) || + (aliasText !== null && aliasName !== null && !aliasMatches) + ) { + coverage.relationBindings = "partial"; + addIssue(issues, "unsupported-relation-source", { + from: site.from, + to: site.to, + }); + } + const nested = derivedAst(rawRelation); + let source: SqlRelationBinding["source"]; + if (nested !== null) { + const nestedId = emitted.ids.get(nested); + if (nestedId === undefined || !site.derived) { + coverage.relationBindings = "partial"; + source = Object.freeze({ + kind: "unknown", + reason: "unsupported-relation-source", + }); + } else { + source = Object.freeze({ block: nestedId, kind: "derived" }); + } + } else { + const path = pathFromAst( + rawRelation, + options.grammar, + site, + ); + if (path === null || site.derived) { + coverage.relationBindings = "partial"; + addIssue(issues, "unsupported-relation-source", { + from: site.from, + to: site.to, + }); + source = Object.freeze({ + kind: "unknown", + reason: "unsupported-relation-source", + }); + } else { + const last = phaseValue(path, path.length - 1); + const environment = declarations.get(block.ast.node); + const key = identifierKey(last); + const declaration = environment?.visible.get(key); + if (declaration !== undefined) { + source = Object.freeze({ + declarationRange: declaration.range, + kind: "cte", + name: declaration.name, + }); + } else if (environment?.uncertain.has(key)) { + coverage.relationBindings = "partial"; + addIssue(issues, "recursive-cte-uncertainty", { + from: site.from, + to: site.to, + }); + source = Object.freeze({ + kind: "unknown", + reason: "unknown-correlation", + }); + } else { + source = Object.freeze({ kind: "named", path }); + } + } + } + const bindingIndex = bindings.length; + bindings.push(Object.freeze({ + alias, + owner: block.id, + range: Object.freeze({ from: site.from, to: site.to }), + source, + })); + scopes.push(Object.freeze({ + addedBinding: bindingIndex, + parentScope: currentScope, + })); + if (!site.derived) { + regions.push(Object.freeze({ + block: block.id, + kind: "from-source", + range: Object.freeze({ from: site.from, to: site.to }), + scope: currentScope, + })); + } + currentScope = scopes.length - 1; + after.push(currentScope); + } + regions.push(...clauseRegions(tokens, block, currentScope)); + regions.push(...joinRegions(tokens, block, sites, after)); + } + + regions.sort((left, right) => + left.range.from - right.range.from || + left.range.to - right.range.to + ); + const nonOverlapping: SqlVisibilityRegion[] = []; + let end = 0; + for (const region of regions) { + if (region.range.from < end) { + coverage.visibility = "partial"; + addIssue(issues, "unsupported-clause", region.range); + continue; + } + nonOverlapping.push(region); + end = region.range.to; + } + + try { + return Object.freeze({ + model: createSqlQueryBindingModel( + statementText, + authority, + { + bindings, + blocks: publicBlocks, + coverage, + issues, + regions: nonOverlapping, + scopes, + statementRange, + }, + ), + status: "ready", + }); + } catch { + return Object.freeze({ reason: "unsupported-shape", status: "unavailable" }); + } +} diff --git a/src/vnext/node-sql-parser-wire.ts b/src/vnext/node-sql-parser-wire.ts index 93cc22d..40613c0 100644 --- a/src/vnext/node-sql-parser-wire.ts +++ b/src/vnext/node-sql-parser-wire.ts @@ -2,9 +2,15 @@ import { MAX_NODE_SQL_PARSER_STATEMENT_LENGTH, type NodeSqlParserBackendOutcome, } from "./node-sql-parser-backend.js"; +import { + createSqlQueryBindingModel, + isSqlQueryBindingModel, + type SqlQueryBindingModel, + type SqlQueryBindingModelData, +} from "./query-binding-model.js"; import type { SqlStatementKind } from "./syntax.js"; -export const NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION = 1 as const; +export const NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION = 2 as const; // The parse request is the largest closed protocol shape. const MAX_NODE_SQL_PARSER_WIRE_RECORD_KEYS = 5; @@ -17,7 +23,7 @@ export type NodeSqlParserWireFailureCode = | "module-load"; export interface NodeSqlParserWireRequest { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "parse"; readonly requestId: number; readonly grammar: NodeSqlParserWireGrammar; @@ -26,34 +32,35 @@ export interface NodeSqlParserWireRequest { export type NodeSqlParserWireMessage = | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "ready"; } | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "parsed"; + readonly queryBindings: SqlQueryBindingModelData | null; readonly requestId: number; readonly statementKind: SqlStatementKind; } | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "syntax-rejected"; readonly requestId: number; } | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "unsupported"; readonly requestId: number; readonly reason: "multiple-statements" | "resource-limit"; } | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "failed"; readonly requestId: number; readonly code: NodeSqlParserWireFailureCode; } | { - readonly protocolVersion: 1; + readonly protocolVersion: 2; readonly kind: "protocol-error"; readonly code: "invalid-request"; }; @@ -174,6 +181,62 @@ function isRequestText(value: unknown): value is string { ); } +const wireBindingAuthority = Object.freeze({}); + +type DecodedWireBindings = + | { readonly valid: false } + | { + readonly model: SqlQueryBindingModel | null; + readonly valid: true; + }; + +function decodeWireBindings(value: unknown): DecodedWireBindings { + if (value === null) { + return { model: null, valid: true }; + } + if (value === null || typeof value !== "object") { + return { valid: false }; + } + try { + const statementRange = Object.getOwnPropertyDescriptor( + value, + "statementRange", + ); + if ( + statementRange === undefined || + !("value" in statementRange) || + statementRange.value === null || + typeof statementRange.value !== "object" + ) { + return { valid: false }; + } + const to = Object.getOwnPropertyDescriptor( + statementRange.value, + "to", + ); + if ( + to === undefined || + !("value" in to) || + typeof to.value !== "number" || + !Number.isSafeInteger(to.value) || + to.value <= 0 || + to.value > MAX_NODE_SQL_PARSER_STATEMENT_LENGTH + ) { + return { valid: false }; + } + return { + model: createSqlQueryBindingModel( + " ".repeat(to.value), + wireBindingAuthority, + value, + ), + valid: true, + }; + } catch { + return { valid: false }; + } +} + function requireRequestId(value: number): void { if (!isRequestId(value)) { throw new TypeError( @@ -299,6 +362,7 @@ export function encodeNodeSqlParserWireProtocolError(): Extract< export function encodeNodeSqlParserWireBackendOutcome( requestId: number, outcome: NodeSqlParserBackendOutcome, + queryBindings: SqlQueryBindingModel | null = null, ): Exclude< NodeSqlParserWireMessage, { readonly kind: "protocol-error" | "ready" } @@ -307,9 +371,18 @@ export function encodeNodeSqlParserWireBackendOutcome( switch (outcome.kind) { case "parsed": requireStatementKind(outcome.statementKind); + if ( + queryBindings !== null && + !isSqlQueryBindingModel(queryBindings) + ) { + throw new TypeError( + "node-sql-parser wire query bindings must be authenticated", + ); + } return Object.freeze({ kind: "parsed", protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, + queryBindings, requestId, statementKind: outcome.statementKind, }); @@ -362,21 +435,27 @@ export function decodeNodeSqlParserWireMessage( : null; case "parsed": { const statementKind = record.values.get("statementKind"); + const queryBindings = decodeWireBindings( + record.values.get("queryBindings"), + ); if ( !hasExactKeys(record, [ "protocolVersion", "kind", + "queryBindings", "requestId", "statementKind", ]) || !isRequestId(requestId) || - !isStatementKind(statementKind) + !isStatementKind(statementKind) || + !queryBindings.valid ) { return null; } return Object.freeze({ kind: "parsed", protocolVersion: NODE_SQL_PARSER_WIRE_PROTOCOL_VERSION, + queryBindings: queryBindings.model, requestId, statementKind, }); diff --git a/src/vnext/query-binding-model.ts b/src/vnext/query-binding-model.ts new file mode 100644 index 0000000..8b3537b --- /dev/null +++ b/src/vnext/query-binding-model.ts @@ -0,0 +1,1171 @@ +import type { + SqlIdentifierComponent, + SqlIdentifierPath, + SqlTextRange, +} from "./types.js"; + +export const MAX_QUERY_BINDING_STATEMENT_LENGTH = 16 * 1024; +export const MAX_QUERY_BINDING_BLOCKS = 256; +export const MAX_QUERY_BINDINGS = 1_024; +export const MAX_QUERY_BINDING_SCOPES = 2_048; +export const MAX_QUERY_VISIBILITY_REGIONS = 2_048; +export const MAX_QUERY_BINDING_ISSUES = 256; +export const MAX_QUERY_BINDING_PATH_COMPONENTS = 8; +export const MAX_QUERY_BINDING_IDENTIFIER_LENGTH = 256; + +export type SqlQueryBindingCoverage = "complete" | "partial"; + +export interface SqlQueryBindingCoverageSet { + readonly queryBlocks: SqlQueryBindingCoverage; + readonly relationBindings: SqlQueryBindingCoverage; + readonly visibility: SqlQueryBindingCoverage; +} + +export type SqlQueryBindingIssueCode = + | "ambiguous-alias" + | "duplicate-alias" + | "opaque-template-context" + | "parser-compatibility" + | "recursive-cte-uncertainty" + | "resource-limit" + | "unknown-correlation" + | "unsupported-clause" + | "unsupported-relation-source"; + +export interface SqlQueryBindingIssue { + readonly code: SqlQueryBindingIssueCode; + readonly range: SqlTextRange; +} + +export type SqlQueryBlockKind = "compound" | "select"; + +export interface SqlQueryBlock { + readonly baseScope: number; + readonly kind: SqlQueryBlockKind; + readonly parentBlock: number | null; + readonly range: SqlTextRange; +} + +export interface SqlRelationScope { + readonly addedBinding: number | null; + readonly parentScope: number | null; +} + +export type SqlVisibilityRegionKind = + | "from-source" + | "group-by" + | "having" + | "join-condition" + | "limit" + | "order-by" + | "other" + | "qualify" + | "select-list" + | "where"; + +export interface SqlVisibilityRegion { + readonly block: number; + readonly kind: SqlVisibilityRegionKind; + readonly range: SqlTextRange; + readonly scope: number; +} + +export interface SqlNamedRelationSource { + readonly kind: "named"; + readonly path: SqlIdentifierPath; +} + +export interface SqlDerivedRelationSource { + readonly block: number; + readonly kind: "derived"; +} + +export interface SqlCteRelationSource { + readonly declarationRange: SqlTextRange; + readonly kind: "cte"; + readonly name: SqlIdentifierComponent; +} + +export interface SqlUnknownRelationSource { + readonly kind: "unknown"; + readonly reason: + | "opaque-template-context" + | "unknown-correlation" + | "unsupported-relation-source"; +} + +export type SqlRelationBindingSource = + | SqlCteRelationSource + | SqlDerivedRelationSource + | SqlNamedRelationSource + | SqlUnknownRelationSource; + +export interface SqlRelationBindingAlias { + readonly explicit: boolean; + readonly name: SqlIdentifierComponent; + readonly range: SqlTextRange; +} + +export interface SqlRelationBinding { + readonly alias: SqlRelationBindingAlias | null; + readonly owner: number; + readonly range: SqlTextRange; + readonly source: SqlRelationBindingSource; +} + +/** Validated plain-data shape used before exact source/authority authentication. */ +export interface SqlQueryBindingModelData { + readonly blocks: readonly SqlQueryBlock[]; + readonly bindings: readonly SqlRelationBinding[]; + readonly coverage: SqlQueryBindingCoverageSet; + readonly issues: readonly SqlQueryBindingIssue[]; + readonly regions: readonly SqlVisibilityRegion[]; + readonly scopes: readonly SqlRelationScope[]; + readonly statementRange: SqlTextRange; +} + +/** A model authenticated for one exact statement and parser authority. */ +export interface SqlQueryBindingModel extends SqlQueryBindingModelData {} + +export type SqlQueryBindingModelErrorCode = + | "invalid-authority" + | "invalid-model" + | "invalid-source" + | "resource-limit"; + +const modelErrors = new WeakSet(); +const models = new WeakSet(); +const modelOrigins = new WeakMap< + object, + { readonly authority: object; readonly statementText: string } +>(); + +export class SqlQueryBindingModelError extends Error { + readonly code: SqlQueryBindingModelErrorCode; + + constructor(code: SqlQueryBindingModelErrorCode, message: string) { + super(message); + this.name = "SqlQueryBindingModelError"; + this.code = code; + modelErrors.add(this); + } +} + +export function isSqlQueryBindingModelError( + candidate: unknown, +): candidate is SqlQueryBindingModelError { + return ( + candidate !== null && + typeof candidate === "object" && + modelErrors.has(candidate) + ); +} + +export function isSqlQueryBindingModel( + candidate: unknown, +): candidate is SqlQueryBindingModel { + return ( + candidate !== null && + typeof candidate === "object" && + models.has(candidate) + ); +} + +export function sqlQueryBindingModelMatches( + model: unknown, + statementText: string, + authority: unknown, +): boolean { + if (model === null || typeof model !== "object") { + return false; + } + const origin = modelOrigins.get(model); + return ( + origin !== undefined && + origin.authority === authority && + origin.statementText === statementText + ); +} + +interface FoundProperty { + readonly found: true; + readonly value: unknown; +} + +interface MissingProperty { + readonly found: false; +} + +type DataProperty = FoundProperty | MissingProperty; + +function ownDataProperty( + value: object, + key: PropertyKey, + subject: string, +): DataProperty { + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (descriptor === undefined) { + return { found: false }; + } + if (!("value" in descriptor)) { + throw invalid(`${subject}.${String(key)} cannot be an accessor`); + } + return { found: true, value: descriptor.value }; +} + +function requiredProperty( + value: object, + key: PropertyKey, + subject: string, +): unknown { + const property = ownDataProperty(value, key, subject); + if (!property.found) { + throw invalid(`${subject}.${String(key)} is required`); + } + return property.value; +} + +function objectValue(value: unknown, subject: string): object { + if (value === null || typeof value !== "object") { + throw invalid(`${subject} must be an object`); + } + return value; +} + +function invalid(message: string): SqlQueryBindingModelError { + return new SqlQueryBindingModelError("invalid-model", message); +} + +function resource(message: string): SqlQueryBindingModelError { + return new SqlQueryBindingModelError("resource-limit", message); +} + +function integer( + value: unknown, + subject: string, + maximum: number, +): number { + if ( + typeof value !== "number" || + !Number.isSafeInteger(value) || + value < 0 || + value >= maximum + ) { + throw invalid(`${subject} must be an in-bounds non-negative integer`); + } + return value; +} + +function nullableInteger( + value: unknown, + subject: string, + maximum: number, +): number | null { + return value === null ? null : integer(value, subject, maximum); +} + +function booleanValue(value: unknown, subject: string): boolean { + if (typeof value !== "boolean") { + throw invalid(`${subject} must be a boolean`); + } + return value; +} + +function enumValue( + value: unknown, + subject: string, + allowed: readonly Value[], +): Value { + if (typeof value !== "string") { + throw invalid(`${subject} is not supported`); + } + const match = allowed.find((candidate) => candidate === value); + if (match === undefined) { + throw invalid(`${subject} is not supported`); + } + return match; +} + +function arrayValues( + value: unknown, + subject: string, + maximum: number, +): readonly unknown[] { + if (!Array.isArray(value)) { + throw invalid(`${subject} must be an array`); + } + const lengthProperty = ownDataProperty(value, "length", subject); + if ( + !lengthProperty.found || + typeof lengthProperty.value !== "number" || + !Number.isSafeInteger(lengthProperty.value) || + lengthProperty.value < 0 + ) { + throw invalid(`${subject} has an invalid length`); + } + if (lengthProperty.value > maximum) { + throw resource(`${subject} exceeds its ${maximum} item limit`); + } + const result: unknown[] = []; + for (let index = 0; index < lengthProperty.value; index += 1) { + const item = ownDataProperty(value, index, subject); + if (!item.found) { + throw invalid(`${subject} cannot contain holes`); + } + result.push(item.value); + } + return result; +} + +function rangeValue( + value: unknown, + statementLength: number, + subject: string, + allowEmpty = true, +): SqlTextRange { + const object = objectValue(value, subject); + const from = integer( + requiredProperty(object, "from", subject), + `${subject}.from`, + statementLength + 1, + ); + const to = integer( + requiredProperty(object, "to", subject), + `${subject}.to`, + statementLength + 1, + ); + if (from > to || (!allowEmpty && from === to)) { + throw invalid(`${subject} must be a valid half-open range`); + } + return Object.freeze({ from, to }); +} + +function rangeContains( + outer: SqlTextRange, + inner: SqlTextRange, +): boolean { + return outer.from <= inner.from && inner.to <= outer.to; +} + +function authenticatedItem( + values: readonly Value[], + index: number, +): Value { + const value = values[index]; + if (value === undefined) { + throw new Error("Authenticated query binding model invariant was violated"); + } + return value; +} + +interface RelationshipValidationIndex { + readonly ancestors: readonly Uint8Array[]; + readonly scopesIncludingLocal: readonly Uint8Array[]; + readonly scopesWithoutLocal: readonly Uint8Array[]; +} + +function relationshipValidationIndex( + model: SqlQueryBindingModel, +): RelationshipValidationIndex { + const ancestors = model.blocks.map((_block, blockIndex) => { + const result = new Uint8Array(model.blocks.length); + let cursor: number | null = blockIndex; + while (cursor !== null) { + result[cursor] = 1; + cursor = authenticatedItem(model.blocks, cursor).parentBlock; + } + return result; + }); + const scopesIncludingLocal: Uint8Array[] = []; + const scopesWithoutLocal: Uint8Array[] = []; + for (let blockIndex = 0; blockIndex < model.blocks.length; blockIndex += 1) { + const includingLocal = new Uint8Array(model.scopes.length); + const withoutLocal = new Uint8Array(model.scopes.length); + const blockAncestors = authenticatedItem(ancestors, blockIndex); + for (let scopeIndex = 0; scopeIndex < model.scopes.length; scopeIndex += 1) { + const scope = authenticatedItem(model.scopes, scopeIndex); + const parentIncluding = scope.parentScope === null || + includingLocal[scope.parentScope] === 1; + const parentWithout = scope.parentScope === null || + withoutLocal[scope.parentScope] === 1; + if (scope.addedBinding === null) { + includingLocal[scopeIndex] = parentIncluding ? 1 : 0; + withoutLocal[scopeIndex] = parentWithout ? 1 : 0; + continue; + } + const binding = authenticatedItem(model.bindings, scope.addedBinding); + const ownerVisible = blockAncestors[binding.owner] === 1; + includingLocal[scopeIndex] = + parentIncluding && ownerVisible ? 1 : 0; + withoutLocal[scopeIndex] = + parentWithout && ownerVisible && binding.owner !== blockIndex ? 1 : 0; + } + scopesIncludingLocal.push(includingLocal); + scopesWithoutLocal.push(withoutLocal); + } + return { + ancestors, + scopesIncludingLocal, + scopesWithoutLocal, + }; +} + +function wellFormedString(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const trailing = value.charCodeAt(index + 1); + if (!(trailing >= 0xdc00 && trailing <= 0xdfff)) { + return false; + } + index += 1; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return false; + } + } + return true; +} + +function identifierValue( + value: unknown, + subject: string, +): SqlIdentifierComponent { + const object = objectValue(value, subject); + const text = requiredProperty(object, "value", subject); + if ( + typeof text !== "string" || + text.length === 0 || + text.length > MAX_QUERY_BINDING_IDENTIFIER_LENGTH || + !wellFormedString(text) + ) { + throw invalid(`${subject}.value must be a bounded well-formed string`); + } + return Object.freeze({ + quoted: booleanValue( + requiredProperty(object, "quoted", subject), + `${subject}.quoted`, + ), + value: text, + }); +} + +function pathValue( + value: unknown, + subject: string, +): SqlIdentifierPath { + const input = arrayValues( + value, + subject, + MAX_QUERY_BINDING_PATH_COMPONENTS, + ); + if (input.length === 0) { + throw invalid(`${subject} cannot be empty`); + } + return Object.freeze( + input.map((component, index) => + identifierValue(component, `${subject}[${index}]`), + ), + ); +} + +const BLOCK_KINDS: readonly SqlQueryBlockKind[] = Object.freeze([ + "compound", + "select", +]); +const REGION_KINDS: readonly SqlVisibilityRegionKind[] = Object.freeze([ + "from-source", + "group-by", + "having", + "join-condition", + "limit", + "order-by", + "other", + "qualify", + "select-list", + "where", +]); +const ISSUE_CODES: readonly SqlQueryBindingIssueCode[] = Object.freeze([ + "ambiguous-alias", + "duplicate-alias", + "opaque-template-context", + "parser-compatibility", + "recursive-cte-uncertainty", + "resource-limit", + "unknown-correlation", + "unsupported-clause", + "unsupported-relation-source", +]); +const UNKNOWN_REASONS: readonly SqlUnknownRelationSource["reason"][] = + Object.freeze([ + "opaque-template-context", + "unknown-correlation", + "unsupported-relation-source", + ]); + +function coverageValue( + value: unknown, +): SqlQueryBindingCoverageSet { + const object = objectValue(value, "query binding coverage"); + const read = (key: string): SqlQueryBindingCoverage => { + const result = enumValue( + requiredProperty(object, key, "query binding coverage"), + `query binding coverage.${key}`, + ["complete", "partial"], + ); + return result === "complete" ? "complete" : "partial"; + }; + return Object.freeze({ + queryBlocks: read("queryBlocks"), + relationBindings: read("relationBindings"), + visibility: read("visibility"), + }); +} + +function blockValue( + value: unknown, + index: number, + count: number, + scopeCount: number, + statementLength: number, +): SqlQueryBlock { + const subject = `query block ${index}`; + const object = objectValue(value, subject); + const kind = enumValue( + requiredProperty(object, "kind", subject), + `${subject}.kind`, + BLOCK_KINDS, + ); + const parentBlock = nullableInteger( + requiredProperty(object, "parentBlock", subject), + `${subject}.parentBlock`, + count, + ); + if (parentBlock !== null && parentBlock >= index) { + throw invalid(`${subject} parent must precede its child`); + } + return Object.freeze({ + baseScope: integer( + requiredProperty(object, "baseScope", subject), + `${subject}.baseScope`, + scopeCount, + ), + kind: kind === "compound" ? "compound" : "select", + parentBlock, + range: rangeValue( + requiredProperty(object, "range", subject), + statementLength, + `${subject}.range`, + false, + ), + }); +} + +function scopeValue( + value: unknown, + index: number, + count: number, + bindingCount: number, +): SqlRelationScope { + const subject = `relation scope ${index}`; + const object = objectValue(value, subject); + const parentScope = nullableInteger( + requiredProperty(object, "parentScope", subject), + `${subject}.parentScope`, + count, + ); + if (parentScope !== null && parentScope >= index) { + throw invalid(`${subject} parent must precede its child`); + } + return Object.freeze({ + addedBinding: nullableInteger( + requiredProperty(object, "addedBinding", subject), + `${subject}.addedBinding`, + bindingCount, + ), + parentScope, + }); +} + +function aliasValue( + value: unknown, + statementLength: number, + subject: string, +): SqlRelationBindingAlias | null { + if (value === null) { + return null; + } + const object = objectValue(value, subject); + return Object.freeze({ + explicit: booleanValue( + requiredProperty(object, "explicit", subject), + `${subject}.explicit`, + ), + name: identifierValue( + requiredProperty(object, "name", subject), + `${subject}.name`, + ), + range: rangeValue( + requiredProperty(object, "range", subject), + statementLength, + `${subject}.range`, + false, + ), + }); +} + +function sourceValue( + value: unknown, + blockCount: number, + statementLength: number, + subject: string, +): SqlRelationBindingSource { + const object = objectValue(value, subject); + const kind = requiredProperty(object, "kind", subject); + if (kind === "named") { + return Object.freeze({ + kind, + path: pathValue(requiredProperty(object, "path", subject), `${subject}.path`), + }); + } + if (kind === "derived") { + return Object.freeze({ + block: integer( + requiredProperty(object, "block", subject), + `${subject}.block`, + blockCount, + ), + kind, + }); + } + if (kind === "cte") { + return Object.freeze({ + declarationRange: rangeValue( + requiredProperty(object, "declarationRange", subject), + statementLength, + `${subject}.declarationRange`, + false, + ), + kind, + name: identifierValue( + requiredProperty(object, "name", subject), + `${subject}.name`, + ), + }); + } + if (kind === "unknown") { + const reason = enumValue( + requiredProperty(object, "reason", subject), + `${subject}.reason`, + UNKNOWN_REASONS, + ); + if (reason === "opaque-template-context") { + return Object.freeze({ kind, reason }); + } + if (reason === "unknown-correlation") { + return Object.freeze({ kind, reason }); + } + return Object.freeze({ + kind, + reason: "unsupported-relation-source", + }); + } + throw invalid(`${subject}.kind is not supported`); +} + +function bindingValue( + value: unknown, + index: number, + blockCount: number, + statementLength: number, +): SqlRelationBinding { + const subject = `relation binding ${index}`; + const object = objectValue(value, subject); + const range = rangeValue( + requiredProperty(object, "range", subject), + statementLength, + `${subject}.range`, + false, + ); + const alias = aliasValue( + requiredProperty(object, "alias", subject), + statementLength, + `${subject}.alias`, + ); + if (alias !== null && !rangeContains(range, alias.range)) { + throw invalid(`${subject} must contain its alias range`); + } + return Object.freeze({ + alias, + owner: integer( + requiredProperty(object, "owner", subject), + `${subject}.owner`, + blockCount, + ), + range, + source: sourceValue( + requiredProperty(object, "source", subject), + blockCount, + statementLength, + `${subject}.source`, + ), + }); +} + +function regionValue( + value: unknown, + index: number, + blockCount: number, + scopeCount: number, + statementLength: number, +): SqlVisibilityRegion { + const subject = `visibility region ${index}`; + const object = objectValue(value, subject); + const kind = enumValue( + requiredProperty(object, "kind", subject), + `${subject}.kind`, + REGION_KINDS, + ); + return Object.freeze({ + block: integer( + requiredProperty(object, "block", subject), + `${subject}.block`, + blockCount, + ), + kind, + range: rangeValue( + requiredProperty(object, "range", subject), + statementLength, + `${subject}.range`, + false, + ), + scope: integer( + requiredProperty(object, "scope", subject), + `${subject}.scope`, + scopeCount, + ), + }); +} + +function issueValue( + value: unknown, + index: number, + statementLength: number, +): SqlQueryBindingIssue { + const subject = `query binding issue ${index}`; + const object = objectValue(value, subject); + const code = enumValue( + requiredProperty(object, "code", subject), + `${subject}.code`, + ISSUE_CODES, + ); + return Object.freeze({ + code, + range: rangeValue( + requiredProperty(object, "range", subject), + statementLength, + `${subject}.range`, + ), + }); +} + +function validateRelationships(model: SqlQueryBindingModel): void { + const validation = relationshipValidationIndex(model); + for (let index = 0; index < model.blocks.length; index += 1) { + const block = authenticatedItem(model.blocks, index); + if (block.parentBlock !== null) { + const parent = authenticatedItem(model.blocks, block.parentBlock); + if (!rangeContains(parent.range, block.range)) { + throw invalid(`query block ${index} is outside its parent`); + } + } + if ( + authenticatedItem(validation.scopesWithoutLocal, index)[block.baseScope] !== + 1 + ) { + throw invalid( + `query block ${index} base scope contains an unrelated relation binding`, + ); + } + for (let sibling = 0; sibling < index; sibling += 1) { + const other = authenticatedItem(model.blocks, sibling); + const overlaps = + block.range.from < other.range.to && + other.range.from < block.range.to; + if ( + overlaps && + authenticatedItem(validation.ancestors, index)[sibling] !== 1 && + authenticatedItem(validation.ancestors, sibling)[index] !== 1 + ) { + throw invalid(`query block ${index} overlaps an unrelated block`); + } + } + } + for (let index = 0; index < model.bindings.length; index += 1) { + const binding = authenticatedItem(model.bindings, index); + const owner = authenticatedItem(model.blocks, binding.owner); + if (!rangeContains(owner.range, binding.range)) { + throw invalid(`relation binding ${index} is outside its owner`); + } + if (binding.source.kind === "derived") { + const derived = authenticatedItem(model.blocks, binding.source.block); + if ( + derived.parentBlock !== binding.owner || + !rangeContains(binding.range, derived.range) + ) { + throw invalid(`derived relation binding ${index} has an unrelated block`); + } + } + if ( + model.coverage.relationBindings === "complete" && + binding.source.kind === "unknown" + ) { + throw invalid("complete relation coverage cannot contain unknown bindings"); + } + } + const introducedBindings = new Set(); + for (let index = 0; index < model.scopes.length; index += 1) { + const scope = authenticatedItem(model.scopes, index); + if (scope.addedBinding !== null) { + if (introducedBindings.has(scope.addedBinding)) { + throw invalid(`relation binding ${scope.addedBinding} is introduced twice`); + } + introducedBindings.add(scope.addedBinding); + } + } + if (introducedBindings.size !== model.bindings.length) { + throw invalid("every relation binding must be introduced by exactly one scope"); + } + let previousEnd = 0; + for (let index = 0; index < model.regions.length; index += 1) { + const region = authenticatedItem(model.regions, index); + if (region.range.from < previousEnd) { + throw invalid("visibility regions must be ordered and non-overlapping"); + } + const block = authenticatedItem(model.blocks, region.block); + if (!rangeContains(block.range, region.range)) { + throw invalid(`visibility region ${index} is outside its block`); + } + if ( + authenticatedItem( + validation.scopesIncludingLocal, + region.block, + )[region.scope] !== 1 + ) { + throw invalid( + `visibility region ${index} scope contains an unrelated relation binding`, + ); + } + previousEnd = region.range.to; + } +} + +export function createSqlQueryBindingModel( + statementText: unknown, + authority: unknown, + input: unknown, +): SqlQueryBindingModel { + if (typeof statementText !== "string") { + throw new SqlQueryBindingModelError( + "invalid-source", + "query binding statement text must be a string", + ); + } + if ( + statementText.length === 0 || + statementText.length > MAX_QUERY_BINDING_STATEMENT_LENGTH + ) { + throw new SqlQueryBindingModelError( + statementText.length > MAX_QUERY_BINDING_STATEMENT_LENGTH + ? "resource-limit" + : "invalid-source", + `query binding statement must contain 1-${MAX_QUERY_BINDING_STATEMENT_LENGTH} UTF-16 code units`, + ); + } + if (authority === null || typeof authority !== "object") { + throw new SqlQueryBindingModelError( + "invalid-authority", + "query binding parser authority must be an object identity", + ); + } + try { + const object = objectValue(input, "query binding model"); + const blockInputs = arrayValues( + requiredProperty(object, "blocks", "query binding model"), + "query blocks", + MAX_QUERY_BINDING_BLOCKS, + ); + const bindingInputs = arrayValues( + requiredProperty(object, "bindings", "query binding model"), + "relation bindings", + MAX_QUERY_BINDINGS, + ); + const scopeInputs = arrayValues( + requiredProperty(object, "scopes", "query binding model"), + "relation scopes", + MAX_QUERY_BINDING_SCOPES, + ); + const regionInputs = arrayValues( + requiredProperty(object, "regions", "query binding model"), + "visibility regions", + MAX_QUERY_VISIBILITY_REGIONS, + ); + const issueInputs = arrayValues( + requiredProperty(object, "issues", "query binding model"), + "query binding issues", + MAX_QUERY_BINDING_ISSUES, + ); + if (blockInputs.length === 0 || scopeInputs.length === 0) { + throw invalid("query binding model requires a block and a scope"); + } + const statementRange = rangeValue( + requiredProperty(object, "statementRange", "query binding model"), + statementText.length, + "query binding model.statementRange", + false, + ); + if (statementRange.from !== 0 || statementRange.to !== statementText.length) { + throw invalid("statementRange must cover the exact statement text"); + } + const blocks = Object.freeze( + blockInputs.map((block, index) => + blockValue( + block, + index, + blockInputs.length, + scopeInputs.length, + statementText.length, + ), + ), + ); + const bindings = Object.freeze( + bindingInputs.map((binding, index) => + bindingValue( + binding, + index, + blockInputs.length, + statementText.length, + ), + ), + ); + const scopes = Object.freeze( + scopeInputs.map((scope, index) => + scopeValue( + scope, + index, + scopeInputs.length, + bindingInputs.length, + ), + ), + ); + const regions = Object.freeze( + regionInputs.map((region, index) => + regionValue( + region, + index, + blockInputs.length, + scopeInputs.length, + statementText.length, + ), + ), + ); + const issues = Object.freeze( + issueInputs.map((issue, index) => + issueValue(issue, index, statementText.length), + ), + ); + const model: SqlQueryBindingModel = Object.freeze({ + blocks, + bindings, + coverage: coverageValue( + requiredProperty(object, "coverage", "query binding model"), + ), + issues, + regions, + scopes, + statementRange, + }); + validateRelationships(model); + models.add(model); + modelOrigins.set(model, Object.freeze({ authority, statementText })); + return model; + } catch (error) { + if (isSqlQueryBindingModelError(error)) { + throw error; + } + throw new SqlQueryBindingModelError( + "invalid-model", + "query binding model could not be inspected safely", + ); + } +} + +export interface SqlVisibleRelationBindings { + readonly bindings: readonly SqlRelationBinding[]; + readonly coverage: SqlQueryBindingCoverage; + readonly issues: readonly SqlQueryBindingIssue[]; + readonly region: SqlVisibilityRegion; + readonly status: "ready"; +} + +export interface SqlVisibleRelationBindingsUnavailable { + readonly reason: "invalid-model" | "outside-visibility-region"; + readonly status: "unavailable"; +} + +export type SqlVisibleRelationBindingsResult = + | SqlVisibleRelationBindings + | SqlVisibleRelationBindingsUnavailable; + +function regionAt( + regions: readonly SqlVisibilityRegion[], + position: number, +): SqlVisibilityRegion | null { + let low = 0; + let high = regions.length; + while (low < high) { + const middle = low + Math.floor((high - low) / 2); + const region = regions[middle]; + if (!region || region.range.from > position) { + high = middle; + } else { + low = middle + 1; + } + } + const candidate = regions[low - 1]; + return candidate && position < candidate.range.to ? candidate : null; +} + +export function visibleSqlRelationBindingsAt( + model: unknown, + position: number, +): SqlVisibleRelationBindingsResult { + if ( + !isSqlQueryBindingModel(model) || + !Number.isSafeInteger(position) || + position < 0 || + position > model.statementRange.to + ) { + return Object.freeze({ reason: "invalid-model", status: "unavailable" }); + } + const region = regionAt(model.regions, position); + if (region === null) { + return Object.freeze({ + reason: "outside-visibility-region", + status: "unavailable", + }); + } + const reversed: SqlRelationBinding[] = []; + let cursor: number | null = region.scope; + while (cursor !== null) { + const scope: SqlRelationScope = + authenticatedItem(model.scopes, cursor); + if (scope.addedBinding !== null) { + const binding = authenticatedItem(model.bindings, scope.addedBinding); + reversed.push(binding); + } + cursor = scope.parentScope; + } + reversed.reverse(); + const coverage = + model.coverage.relationBindings === "complete" && + model.coverage.visibility === "complete" + ? "complete" + : "partial"; + return Object.freeze({ + bindings: Object.freeze(reversed), + coverage, + issues: model.issues, + region, + status: "ready", + }); +} + +export type SqlIdentifierComparator = ( + left: SqlIdentifierComponent, + right: SqlIdentifierComponent, +) => boolean; + +export interface SqlQualifierResolutionFound { + readonly binding: SqlRelationBinding; + readonly coverage: SqlQueryBindingCoverage; + readonly status: "resolved"; +} + +export interface SqlQualifierResolutionAmbiguous { + readonly bindings: readonly SqlRelationBinding[]; + readonly coverage: SqlQueryBindingCoverage; + readonly status: "ambiguous"; +} + +export interface SqlQualifierResolutionMissing { + readonly status: "no-match"; +} + +export interface SqlQualifierResolutionUnavailable { + readonly reason: + | "invalid-model" + | "outside-visibility-region" + | "partial-coverage"; + readonly status: "unavailable"; +} + +export type SqlQualifierResolution = + | SqlQualifierResolutionAmbiguous + | SqlQualifierResolutionFound + | SqlQualifierResolutionMissing + | SqlQualifierResolutionUnavailable; + +function bindingVisibleName( + binding: SqlRelationBinding, +): SqlIdentifierComponent | null { + if (binding.alias !== null) { + return binding.alias.name; + } + if (binding.source.kind === "named") { + return binding.source.path[binding.source.path.length - 1] ?? null; + } + if (binding.source.kind === "cte") { + return binding.source.name; + } + return null; +} + +export function resolveSqlRelationQualifier( + model: unknown, + position: number, + qualifier: SqlIdentifierComponent, + equals: SqlIdentifierComparator, +): SqlQualifierResolution { + const visible = visibleSqlRelationBindingsAt(model, position); + if (visible.status === "unavailable") { + return visible; + } + const matches: SqlRelationBinding[] = []; + for (const binding of visible.bindings) { + const name = bindingVisibleName(binding); + if (name !== null && equals(name, qualifier)) { + matches.push(binding); + } + } + if (matches.length === 0) { + return visible.coverage === "complete" + ? Object.freeze({ status: "no-match" }) + : Object.freeze({ + reason: "partial-coverage", + status: "unavailable", + }); + } + if (matches.length === 1) { + const binding = authenticatedItem(matches, 0); + return Object.freeze({ + binding, + coverage: visible.coverage, + status: "resolved", + }); + } + return Object.freeze({ + bindings: Object.freeze(matches), + coverage: visible.coverage, + status: "ambiguous", + }); +} diff --git a/src/vnext/relation-completion-types.ts b/src/vnext/relation-completion-types.ts index ce950fd..c6057fe 100644 --- a/src/vnext/relation-completion-types.ts +++ b/src/vnext/relation-completion-types.ts @@ -249,6 +249,23 @@ export interface SqlCatalogCompletionProvenance { readonly entityId: string; } +export interface SqlColumnCompletionProvenance { + readonly kind: "column-catalog"; + readonly providerId: string; + readonly scope: string; + readonly epoch: SqlCatalogEpoch; + readonly relationEntityId: string; + readonly columnEntityId: string; +} + +export interface SqlNamespaceCompletionProvenance { + readonly containerEntityId: string; + readonly epoch: SqlCatalogEpoch; + readonly kind: "namespace-catalog"; + readonly providerId: string; + readonly scope: string; +} + interface SqlCompletionItemBase { readonly label: string; readonly edit: SqlTextChange; @@ -265,6 +282,17 @@ export type SqlCompletionItem = readonly kind: "relation"; readonly relationKind: SqlCatalogRelationKind; readonly provenance: SqlCatalogCompletionProvenance; + }) + | (SqlCompletionItemBase & { + readonly dataType?: string; + readonly kind: "column"; + readonly provenance: SqlColumnCompletionProvenance; + readonly relationRequestKey: string; + }) + | (SqlCompletionItemBase & { + readonly kind: "namespace"; + readonly provenance: SqlNamespaceCompletionProvenance; + readonly role: SqlCatalogContainerRole; }); export type SqlCompletionIssue = @@ -272,6 +300,12 @@ export type SqlCompletionIssue = readonly reason: "catalog-loading"; readonly remainingIntentLeaseMs: number; } + | { + readonly reason: + | "column-catalog-loading" + | "namespace-catalog-loading"; + readonly remainingIntentLeaseMs?: number; + } | { readonly reason: | "catalog-partial" @@ -281,7 +315,15 @@ export type SqlCompletionIssue = | "catalog-overloaded" | "catalog-queue-timeout" | "catalog-timeout" + | "column-catalog-failed" + | "column-catalog-malformed" + | "column-catalog-partial" | "cte-scope-uncertainty" + | "namespace-catalog-failed" + | "namespace-catalog-malformed" + | "namespace-catalog-partial" + | "namespace-prefix-uncertain" + | "query-binding-partial" | "query-site-recovery" | "opaque-template-context" | "recursive-cte-uncertainty" @@ -345,6 +387,81 @@ export type SqlCatalogProviderReport = readonly reason: SqlCatalogProviderUnavailableReason; }); +export type SqlColumnCatalogProviderReport = + | { + readonly feature: "column-catalog"; + readonly outcome: "ready"; + readonly providerId: string; + readonly coverage: "complete" | "partial"; + readonly failures: readonly SqlColumnCatalogFailure[]; + } + | { + readonly feature: "column-catalog"; + readonly outcome: "loading"; + readonly providerId: string; + readonly failures: readonly SqlColumnCatalogFailure[]; + } + | { + readonly feature: "column-catalog"; + readonly outcome: "failed"; + readonly providerId: string; + readonly failures: readonly [ + SqlColumnCatalogFailure, + ...SqlColumnCatalogFailure[], + ]; + } + | { + readonly feature: "column-catalog"; + readonly outcome: "unavailable"; + readonly providerId: string; + readonly reason: + | "disposed" + | "invalid-request" + | "malformed-response" + | "provider-failed"; + }; + +export interface SqlColumnCatalogFailure { + readonly code: SqlCatalogFailureCode; + readonly requestKey: string; + readonly retry: SqlCatalogRetryPolicy; +} + +export type SqlNamespaceCatalogProviderReport = + | { + readonly coverage: "complete" | "partial"; + readonly feature: "namespace-catalog"; + readonly outcome: "ready"; + readonly providerId: string; + } + | { + readonly feature: "namespace-catalog"; + readonly outcome: "loading"; + readonly providerId: string; + } + | { + readonly feature: "namespace-catalog"; + readonly outcome: "failed"; + readonly providerId: string; + readonly code: SqlCatalogFailureCode; + readonly retry: SqlCatalogRetryPolicy; + } + | { + readonly feature: "namespace-catalog"; + readonly outcome: "unavailable"; + readonly providerId: string; + readonly reason: + | "disposed" + | "invalid-request" + | "malformed-response" + | "provider-failed"; + }; + +export type SqlCompletionProviderReport = + | SqlCatalogProviderReport + | SqlColumnCatalogProviderReport + | SqlNamespaceCatalogProviderReport; + export interface SqlServiceFailure { readonly code: "internal"; readonly retryable: boolean; @@ -356,7 +473,7 @@ export type SqlCompletionResult = readonly revision: SqlRevision; readonly refreshToken: SqlCompletionRefreshToken | null; readonly value: SqlCompletionList; - readonly sources: readonly SqlCatalogProviderReport[]; + readonly sources: readonly SqlCompletionProviderReport[]; } | { readonly status: "unavailable"; diff --git a/src/vnext/relation-completion.ts b/src/vnext/relation-completion.ts index a0d2668..87ac3cb 100644 --- a/src/vnext/relation-completion.ts +++ b/src/vnext/relation-completion.ts @@ -72,7 +72,7 @@ interface RankedCteItem { interface RankedCatalogItem { readonly completionPathLength: number; readonly item: Exclude< - SqlCompletionItem, + Extract, { readonly relationKind: "cte" } >; readonly label: string; diff --git a/src/vnext/session.ts b/src/vnext/session.ts index bb5359d..628410a 100644 --- a/src/vnext/session.ts +++ b/src/vnext/session.ts @@ -10,6 +10,31 @@ import type { SqlTextChange, SqlTextRange, } from "./types.js"; +import { + createSqlColumnCatalogBatchCoordinator, + type SqlColumnCatalogBatchCoordinator, + type SqlColumnCatalogBatchOwner, + type SqlColumnCatalogBatchTicket, +} from "./column-catalog-batch-coordinator.js"; +import { + composeSqlColumnCompletion, + prepareSqlColumnCatalogRelations, +} from "./column-completion.js"; +import { + createSqlNamespaceCatalogCoordinator, + type SqlNamespaceCatalogCoordinator, + type SqlNamespaceCatalogOwner, + type SqlNamespaceCatalogSearchOutcome, + type SqlNamespaceCatalogSearchTicket, +} from "./namespace-catalog-coordinator.js"; +import { + composeSqlNamespaceCompletion, + prepareSqlNamespaceCatalogSearch, + type SqlNamespaceCompletionComposition, +} from "./namespace-completion.js"; +import { + MAX_NAMESPACE_RESULTS, +} from "./namespace-catalog-boundary.js"; import { buildSqlStatementIndex, findSqlStatementSlot, @@ -34,6 +59,7 @@ import { type SqlCatalogSearchWorkTicket, } from "./relation-catalog-search-work.js"; import { + analyzeSqlLocalColumnSite, analyzeSqlLocalRelationSite, prepareSqlLocalRelationStatement, type SqlLocalRelationStatementPreparation, @@ -48,6 +74,9 @@ import { createSqlCompletionRefreshToken, type SqlCompletionRequest, type SqlCompletionRefreshToken, + type SqlCompletionIssue, + type SqlCompletionItem, + type SqlCompletionList, type SqlDisposable, type SqlCompletionResult, type SqlCompletionTask, @@ -95,6 +124,7 @@ const MAX_DIALECTS = 1_000; const DEFAULT_CATALOG_RESPONSE_BUDGET_MS = 40; const MAX_CATALOG_RESPONSE_BUDGET_MS = 50; const TERMINAL_LOADING_INTENT_LEASE_MS = 1_000; +const AUXILIARY_LOADING_RETRY_DELAY_MS = 100; interface ResolvedCatalogContext { readonly scope: string; @@ -104,7 +134,11 @@ interface ResolvedCatalogContext { interface CompletionRequestState { cancelReason: "caller" | "disposed" | "superseded" | null; readonly revision: SqlRevision; - ticket: SqlCatalogSearchWorkTicket | null; + readonly tickets: Set< + | SqlCatalogSearchWorkTicket + | SqlColumnCatalogBatchTicket + | SqlNamespaceCatalogSearchTicket + >; readonly token: SqlCompletionRefreshToken; } @@ -132,6 +166,12 @@ interface TerminalRefreshIntent { readonly token: SqlCompletionRefreshToken; } +interface AuxiliaryLoadingRetry { + readonly context: Context; + readonly position: number; + readonly source: SqlSourceSnapshot; +} + interface CompletionConfiguration { readonly catalogResponseBudgetMs: number; } @@ -531,6 +571,130 @@ function completionCancellationReason( return request.cancelReason ?? "superseded"; } +function cancelCompletionTickets( + request: CompletionRequestState | null, +): void { + if (!request) return; + for (const ticket of request.tickets) ticket.cancel(); +} + +async function raceCatalogResponse( + result: Promise, + timeoutMs: number, +): Promise< + | { readonly kind: "outcome"; readonly outcome: Outcome } + | { readonly kind: "timeout" } +> { + let timer: ReturnType | undefined; + const raced = await Promise.race([ + result.then((outcome) => + Object.freeze({ kind: "outcome" as const, outcome }) + ), + new Promise<{ readonly kind: "timeout" }>((resolve) => { + timer = setTimeout( + () => resolve(Object.freeze({ kind: "timeout" })), + timeoutMs, + ); + }), + ]); + clearTimeout(timer); + return raced; +} + +function remainingCatalogBudget( + startedAt: number, + budgetMs: number, +): number { + return Math.max(0, budgetMs - Math.max(0, performance.now() - startedAt)); +} + +function namespaceCompletionList( + composition: SqlNamespaceCompletionComposition, +): SqlCompletionList { + const items: readonly SqlCompletionItem[] = + composition.value.items.map((item) => + Object.freeze({ + ...(item.detail === undefined ? {} : { detail: item.detail }), + edit: item.edit, + kind: "namespace" as const, + label: item.label, + provenance: item.provenance, + role: item.role, + }) + ); + const issues: readonly SqlCompletionIssue[] = + composition.value.issues.map((reason) => + Object.freeze({ reason }) + ); + const first = issues[0]; + if (first === undefined) { + return Object.freeze({ + isIncomplete: false, + issues: Object.freeze([] as const), + items: Object.freeze(items), + }); + } + const incompleteIssues: [ + SqlCompletionIssue, + ...SqlCompletionIssue[], + ] = [first, ...issues.slice(1)]; + return Object.freeze({ + isIncomplete: true, + issues: Object.freeze(incompleteIssues), + items: Object.freeze(items), + }); +} + +function mergeCompletionLists( + left: SqlCompletionList, + right: SqlCompletionList, +): SqlCompletionList { + const items = Object.freeze([...left.items, ...right.items]); + const issues = Object.freeze([...left.issues, ...right.issues]); + const first = issues[0]; + if (first === undefined) { + return Object.freeze({ + isIncomplete: false, + issues: Object.freeze([] as const), + items, + }); + } + const incompleteIssues: [ + SqlCompletionIssue, + ...SqlCompletionIssue[], + ] = [first, ...issues.slice(1)]; + return Object.freeze({ + isIncomplete: true, + issues: Object.freeze(incompleteIssues), + items, + }); +} + +function completionListWithLoadingLease( + value: SqlCompletionList, + reason: + | "column-catalog-loading" + | "namespace-catalog-loading", + remainingIntentLeaseMs: number, +): SqlCompletionList { + const issues = value.issues.map((issue) => + issue.reason === reason + ? Object.freeze({ reason, remainingIntentLeaseMs }) + : issue + ); + const first = issues[0]; + if (!first) return value; + const incompleteIssues: [ + SqlCompletionIssue, + ...SqlCompletionIssue[], + ] = [first, ...issues.slice(1)]; + return Object.freeze({ + isIncomplete: true, + issues: Object.freeze(incompleteIssues), + items: value.items, + }); +} + interface MissingDataProperty { readonly found: false; } @@ -937,13 +1101,23 @@ export class DefaultSqlDocumentSession { readonly #catalogCoordinator: SqlCatalogSearchWorkCoordinator | null; readonly #catalogResponseBudgetMs: number; + readonly #columnCoordinator: SqlColumnCatalogBatchCoordinator | null; + readonly #namespaceCoordinator: SqlNamespaceCatalogCoordinator | null; readonly #dialects: ReadonlyMap; readonly #onDispose: () => void; readonly #listeners = new Set(); #activeCompletion: CompletionRequestState | null = null; + #columnLoadingRetry: AuxiliaryLoadingRetry | null = null; #catalogOwner: SqlCatalogSearchWorkOwner | null = null; #catalogOwnerDialect: SqlRelationDialectRuntime | null = null; #catalogOwnerScope: string | null = null; + #columnOwner: SqlColumnCatalogBatchOwner | null = null; + #columnOwnerDialect: SqlRelationDialectRuntime | null = null; + #columnOwnerScope: string | null = null; + #namespaceOwner: SqlNamespaceCatalogOwner | null = null; + #namespaceOwnerDialect: SqlRelationDialectRuntime | null = null; + #namespaceOwnerScope: string | null = null; + #namespaceLoadingRetry: AuxiliaryLoadingRetry | null = null; #disposed = false; #localRelationStatementCache: | LocalRelationStatementCache @@ -960,10 +1134,14 @@ export class DefaultSqlDocumentSession context: Context, dialects: ReadonlyMap, catalogCoordinator: SqlCatalogSearchWorkCoordinator | null, + columnCoordinator: SqlColumnCatalogBatchCoordinator | null, + namespaceCoordinator: SqlNamespaceCatalogCoordinator | null, completion: CompletionConfiguration, onDispose: () => void, ) { this.#catalogCoordinator = catalogCoordinator; + this.#columnCoordinator = columnCoordinator; + this.#namespaceCoordinator = namespaceCoordinator; this.#catalogResponseBudgetMs = completion.catalogResponseBudgetMs; this.#dialects = dialects; @@ -984,6 +1162,8 @@ export class DefaultSqlDocumentSession sourceSequence, }); this.#replaceCatalogOwner(); + this.#replaceColumnOwner(); + this.#replaceNamespaceOwner(); } get revision(): SqlRevision { @@ -1100,6 +1280,77 @@ export class DefaultSqlDocumentSession clearTimeout(timer.handle); } + #retainAuxiliaryRefresh( + active: CompletionRequestState, + readiness: Promise, + ): number { + this.#clearSoftRefreshIntentTimer(); + this.#refreshIntent = active; + const cell: SessionTimerCell = { + active: true, + handle: undefined, + }; + this.#softRefreshIntentTimer = cell; + const expire = setTimeout(() => { + cell.active = false; + this.#softRefreshIntentTimer = null; + this.#refreshIntent = null; + cancelCompletionTickets(active); + }, TERMINAL_LOADING_INTENT_LEASE_MS); + cell.handle = expire; + void readiness.then( + () => { + if ( + !cell.active || + this.#softRefreshIntentTimer !== cell + ) { + return; + } + const commit = this.#prepareServiceChange({ + expected: active, + reason: "catalog-availability", + }); + commit?.(); + }, + () => { + const commit = this.#prepareServiceChange({ + expected: active, + reason: "catalog-availability", + }); + commit?.(); + }, + ); + return TERMINAL_LOADING_INTENT_LEASE_MS; + } + + #claimAuxiliaryLoadingRetry( + feature: "column" | "namespace", + snapshot: SessionSnapshot, + position: number, + ): boolean { + const previous = feature === "column" + ? this.#columnLoadingRetry + : this.#namespaceLoadingRetry; + if ( + previous?.context === snapshot.context && + previous.source === snapshot.source && + previous.position === position + ) { + return false; + } + const next = Object.freeze({ + context: snapshot.context, + position, + source: snapshot.source, + }); + if (feature === "column") { + this.#columnLoadingRetry = next; + } else { + this.#namespaceLoadingRetry = next; + } + return true; + } + #dispatchChange(event: SqlSessionChangeEvent): void { if ( this.#disposed || @@ -1192,8 +1443,22 @@ export class DefaultSqlDocumentSession } } return (): undefined => { - active?.ticket?.cancel(); - if (intent !== active) intent?.ticket?.cancel(); + cancelCompletionTickets(active); + if (intent !== active) cancelCompletionTickets(intent); + if (change.reason === "catalog") { + this.#columnLoadingRetry = null; + this.#namespaceLoadingRetry = null; + this.#columnOwner?.dispose(); + this.#columnOwner = null; + this.#columnOwnerDialect = null; + this.#columnOwnerScope = null; + this.#namespaceOwner?.dispose(); + this.#namespaceOwner = null; + this.#namespaceOwnerDialect = null; + this.#namespaceOwnerScope = null; + this.#replaceColumnOwner(); + this.#replaceNamespaceOwner(); + } this.#dispatchChange(event); return undefined; }; @@ -1240,6 +1505,62 @@ export class DefaultSqlDocumentSession } } + #replaceColumnOwner(): void { + const catalog = resolveCatalogContext(this.#snapshot.context); + const dialect = this.#snapshot.dialect.relationDialect; + if ( + this.#columnOwner && + catalog && + this.#columnOwnerScope === catalog.scope && + this.#columnOwnerDialect === dialect + ) { + return; + } + this.#columnOwner?.dispose(); + this.#columnOwner = null; + this.#columnOwnerDialect = null; + this.#columnOwnerScope = null; + if (!catalog || !this.#columnCoordinator || this.#disposed) { + return; + } + const prepared = this.#columnCoordinator.prepareOwner({ + dialectId: dialect.id, + scope: catalog.scope, + }); + if (prepared.status !== "prepared") return; + this.#columnOwner = prepared.owner; + this.#columnOwnerDialect = dialect; + this.#columnOwnerScope = catalog.scope; + } + + #replaceNamespaceOwner(): void { + const catalog = resolveCatalogContext(this.#snapshot.context); + const dialect = this.#snapshot.dialect.relationDialect; + if ( + this.#namespaceOwner && + catalog && + this.#namespaceOwnerScope === catalog.scope && + this.#namespaceOwnerDialect === dialect + ) { + return; + } + this.#namespaceOwner?.dispose(); + this.#namespaceOwner = null; + this.#namespaceOwnerDialect = null; + this.#namespaceOwnerScope = null; + if (!catalog || !this.#namespaceCoordinator || this.#disposed) { + return; + } + const prepared = this.#namespaceCoordinator.prepareOwner({ + dialectId: dialect.id, + scope: catalog.scope, + }); + if (prepared.status !== "prepared") return; + this.#namespaceOwner = prepared.owner; + this.#namespaceOwnerDialect = dialect; + this.#namespaceOwnerScope = catalog.scope; + } + readonly onDidChange = ( listener: (event: SqlSessionChangeEvent) => void, ): SqlDisposable => { @@ -1269,6 +1590,25 @@ export class DefaultSqlDocumentSession }); }; + readonly invalidateCatalog = (): SqlRevision => { + if (this.#disposed) { + throw new SqlSessionError( + "session-disposed", + "SQL document session is disposed", + ); + } + const commit = this.#prepareServiceChange({ reason: "catalog" }); + if (!commit) { + throw new SqlSessionError( + "session-disposed", + "SQL document session is disposed", + ); + } + const revision = this.#snapshot.revision; + commit(); + return revision; + }; + readonly complete = ( request: SqlCompletionRequest, ): SqlCompletionTask => { @@ -1394,9 +1734,9 @@ export class DefaultSqlDocumentSession previousIntent.cancelReason = "superseded"; } const cancelPrevious = (): void => { - previousActive?.ticket?.cancel(); + cancelCompletionTickets(previousActive); if (previousIntent !== previousActive) { - previousIntent?.ticket?.cancel(); + cancelCompletionTickets(previousIntent); } if (this.#refreshIntent === previousIntent) { this.#refreshIntent = null; @@ -1409,19 +1749,17 @@ export class DefaultSqlDocumentSession const active: CompletionRequestState = { cancelReason: null, revision: snapshot.revision, - ticket: null, + tickets: new Set(), token: refreshToken, }; this.#activeCompletion = active; + const isCurrent = (): boolean => + this.#activeCompletion === active && + active.cancelReason === null && + snapshot.revision === this.#snapshot.revision; const cancellationIfNotCurrent = (): SqlCompletionResult | null => { - if ( - this.#activeCompletion === active && - active.cancelReason === null && - snapshot.revision === this.#snapshot.revision - ) { - return null; - } + if (isCurrent()) return null; cancelPrevious(); return completionCancellation( snapshot.revision, @@ -1437,7 +1775,7 @@ export class DefaultSqlDocumentSession active.cancelReason === null ) { active.cancelReason = "caller"; - active.ticket?.cancel(); + cancelCompletionTickets(active); } }; const makeInvocationInert = (): void => { @@ -1539,6 +1877,135 @@ export class DefaultSqlDocumentSession position, ); if (localSite.status !== "ready") { + const columnSite = analyzeSqlLocalColumnSite( + prepared.statement, + position, + ); + if ( + localSite.status === "inactive" && + columnSite.status === "ready" + ) { + const catalog = resolveCatalogContext(snapshot.context); + const columnOwner = this.#columnOwner; + const columnCoordinator = this.#columnCoordinator; + const preparedRelations = prepareSqlColumnCatalogRelations( + columnSite, + snapshot.dialect.relationDialect, + ); + if (preparedRelations.references.length === 0) { + cancelPrevious(); + return Object.freeze({ + reason: unavailableCompletionReason(localSite), + retryable: false, + revision: snapshot.revision, + status: "unavailable", + }); + } + if (!catalog || !columnOwner || !columnCoordinator) { + cancelPrevious(); + return Object.freeze({ + reason: "unsupported-query-site", + retryable: false, + revision: snapshot.revision, + status: "unavailable", + }); + } + if (!isCurrent()) { + return completionCancellation( + snapshot.revision, + completionCancellationReason(active), + ); + } + const ticket = columnOwner.request({ + expectedEpoch: null, + relations: preparedRelations.references, + searchPaths: catalog.searchPaths, + }); + active.tickets.add(ticket); + const raced = await raceCatalogResponse( + ticket.result, + remainingCatalogBudget( + completionStartedAt, + this.#catalogResponseBudgetMs, + ), + ); + if (!isCurrent()) { + ticket.cancel(); + return completionCancellation( + snapshot.revision, + completionCancellationReason(active), + ); + } + if (raced.kind === "timeout") { + const remainingIntentLeaseMs = + this.#retainAuxiliaryRefresh(active, ticket.result); + return Object.freeze({ + refreshToken: active.token, + revision: snapshot.revision, + sources: Object.freeze([Object.freeze({ + feature: "column-catalog" as const, + failures: Object.freeze([]), + outcome: "loading" as const, + providerId: columnCoordinator.providerId, + })]), + status: "ready", + value: Object.freeze({ + isIncomplete: true, + issues: Object.freeze([Object.freeze({ + reason: "column-catalog-loading" as const, + remainingIntentLeaseMs, + })] as const), + items: Object.freeze([]), + }), + }); + } + const composition = composeSqlColumnCompletion({ + dialect: snapshot.dialect.relationDialect, + outcome: raced.outcome, + prepared: preparedRelations, + providerId: columnCoordinator.providerId, + site: columnSite, + }); + if (!composition) { + return completionCancellation( + snapshot.revision, + completionCancellationReason(active), + ); + } + const columnLoading = + composition.sources[0]?.outcome === "loading"; + const retryLoading = + columnLoading && + this.#claimAuxiliaryLoadingRetry( + "column", + snapshot, + request.position, + ); + const remainingIntentLeaseMs = retryLoading + ? this.#retainAuxiliaryRefresh( + active, + new Promise((resolve) => { + setTimeout( + resolve, + AUXILIARY_LOADING_RETRY_DELAY_MS, + ); + }), + ) + : 0; + return Object.freeze({ + refreshToken: retryLoading ? active.token : null, + revision: snapshot.revision, + sources: composition.sources, + status: "ready", + value: retryLoading + ? completionListWithLoadingLease( + composition.value, + "column-catalog-loading", + remainingIntentLeaseMs, + ) + : composition.value, + }); + } cancelPrevious(); return Object.freeze({ reason: unavailableCompletionReason(localSite), @@ -1547,11 +2014,7 @@ export class DefaultSqlDocumentSession status: "unavailable", }); } - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + if (!isCurrent()) { return completionCancellation( snapshot.revision, completionCancellationReason(active), @@ -1578,11 +2041,7 @@ export class DefaultSqlDocumentSession status: "unavailable", }); } else { - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + if (!isCurrent()) { cancelPrevious(); return completionCancellation( snapshot.revision, @@ -1599,38 +2058,15 @@ export class DefaultSqlDocumentSession if (this.#refreshIntent === previousIntent) { this.#refreshIntent = null; } - active.ticket = ticket; - const elapsed = Math.max( - 0, - performance.now() - completionStartedAt, - ); - const remaining = Math.max( - 0, - this.#catalogResponseBudgetMs - elapsed, - ); - let responseTimer: - | ReturnType - | undefined; - const raced = await Promise.race([ - ticket.result.then((outcome) => - Object.freeze({ - kind: "outcome" as const, - outcome, - }), + active.tickets.add(ticket); + const raced = await raceCatalogResponse( + ticket.result, + remainingCatalogBudget( + completionStartedAt, + this.#catalogResponseBudgetMs, ), - new Promise<{ readonly kind: "timeout" }>((resolve) => { - responseTimer = setTimeout( - () => resolve(Object.freeze({ kind: "timeout" })), - remaining, - ); - }), - ]); - clearTimeout(responseTimer); - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + ); + if (!isCurrent()) { ticket.cancel(); return completionCancellation( snapshot.revision, @@ -1774,6 +2210,95 @@ export class DefaultSqlDocumentSession cancelPrevious(); } + let namespaceComposition: SqlNamespaceCompletionComposition | null = + null; + let namespaceLoadingLeaseMs = 0; + const namespaceOwner = this.#namespaceOwner; + const namespaceCoordinator = this.#namespaceCoordinator; + if (catalog && namespaceOwner && namespaceCoordinator) { + const ticket = namespaceOwner.request( + prepareSqlNamespaceCatalogSearch( + { + prefix: querySite.prefix, + qualifier: querySite.qualifier, + replacementRange, + }, + null, + catalog.searchPaths, + MAX_NAMESPACE_RESULTS, + ), + ); + active.tickets.add(ticket); + const raced = await raceCatalogResponse( + ticket.result, + remainingCatalogBudget( + completionStartedAt, + this.#catalogResponseBudgetMs, + ), + ); + if (!isCurrent()) { + ticket.cancel(); + return completionCancellation( + snapshot.revision, + completionCancellationReason(active), + ); + } + if (raced.kind === "timeout") { + namespaceLoadingLeaseMs = + this.#retainAuxiliaryRefresh(active, ticket.result); + namespaceComposition = Object.freeze({ + source: Object.freeze({ + feature: "namespace-catalog", + outcome: "loading", + providerId: namespaceCoordinator.providerId, + }), + value: Object.freeze({ + isIncomplete: true, + issues: Object.freeze([ + "namespace-catalog-loading" as const, + ] as const), + items: Object.freeze([]), + }), + }); + } else { + const outcome: SqlNamespaceCatalogSearchOutcome = + raced.outcome; + namespaceComposition = composeSqlNamespaceCompletion({ + matchPrefix: + snapshot.dialect.relationDialect.completion + .cteIdentifierMatchesPrefix, + outcome, + prefix: querySite.prefix, + providerId: namespaceCoordinator.providerId, + replacementRange, + }); + if (!namespaceComposition) { + return completionCancellation( + snapshot.revision, + completionCancellationReason(active), + ); + } + if (namespaceComposition.source.outcome === "loading") { + namespaceLoadingLeaseMs = + this.#claimAuxiliaryLoadingRetry( + "namespace", + snapshot, + request.position, + ) + ? this.#retainAuxiliaryRefresh( + active, + new Promise((resolve) => { + setTimeout( + resolve, + AUXILIARY_LOADING_RETRY_DELAY_MS, + ); + }), + ) + : 0; + } + } + } + const composition = composeSqlRelationCompletion({ catalogOutcome, dialect: snapshot.dialect.relationDialect, @@ -1786,19 +2311,34 @@ export class DefaultSqlDocumentSession replacementRange, statementOffset, }); - if ( - this.#activeCompletion !== active || - active.cancelReason !== null || - snapshot.revision !== this.#snapshot.revision - ) { + if (!isCurrent()) { return completionCancellation( snapshot.revision, completionCancellationReason(active), ); } + const value = namespaceComposition + ? mergeCompletionLists( + composition.value, + namespaceLoadingLeaseMs > 0 + ? completionListWithLoadingLease( + namespaceCompletionList(namespaceComposition), + "namespace-catalog-loading", + namespaceLoadingLeaseMs, + ) + : namespaceCompletionList(namespaceComposition), + ) + : composition.value; + const sources = namespaceComposition + ? Object.freeze([ + ...composition.sources, + namespaceComposition.source, + ]) + : composition.sources; return Object.freeze({ refreshToken: - (catalogOutcome?.status === "loading" || + (namespaceLoadingLeaseMs > 0 || + catalogOutcome?.status === "loading" || (catalogOutcome?.status === "usable" && catalogOutcome.response.status === "loading")) && (this.#refreshIntent === active || @@ -1806,9 +2346,9 @@ export class DefaultSqlDocumentSession ? active.token : null, revision: snapshot.revision, - sources: composition.sources, + sources, status: "ready", - value: composition.value, + value, }); } finally { if (this.#activeCompletion === active) { @@ -2029,7 +2569,12 @@ export class DefaultSqlDocumentSession this.#dialects, ); const nextCatalog = resolveCatalogContext(nextContext); - if (nextCatalog && !this.#catalogCoordinator) { + if ( + nextCatalog && + !this.#catalogCoordinator && + !this.#columnCoordinator && + !this.#namespaceCoordinator + ) { throw new SqlSessionError( "invalid-context", "SQL catalog context requires a configured catalog provider", @@ -2100,6 +2645,8 @@ export class DefaultSqlDocumentSession this.#localRelationStatementCache = null; } this.#activeCompletion = null; + this.#columnLoadingRetry = null; + this.#namespaceLoadingRetry = null; this.#refreshIntent = null; this.#clearSoftRefreshIntentTimer(); this.#clearTerminalIntent(); @@ -2116,11 +2663,13 @@ export class DefaultSqlDocumentSession ) { refreshIntent.cancelReason = "superseded"; } - activeCompletion?.ticket?.cancel(); + cancelCompletionTickets(activeCompletion); if (refreshIntent !== activeCompletion) { - refreshIntent?.ticket?.cancel(); + cancelCompletionTickets(refreshIntent); } this.#replaceCatalogOwner(); + this.#replaceColumnOwner(); + this.#replaceNamespaceOwner(); return this.#snapshot.revision; } @@ -2136,9 +2685,15 @@ export class DefaultSqlDocumentSession const activeCompletion = this.#activeCompletion; const refreshIntent = this.#refreshIntent; const catalogOwner = this.#catalogOwner; + const columnOwner = this.#columnOwner; + const namespaceOwner = this.#namespaceOwner; this.#activeCompletion = null; this.#refreshIntent = null; this.#catalogOwner = null; + this.#columnLoadingRetry = null; + this.#columnOwner = null; + this.#namespaceOwner = null; + this.#namespaceLoadingRetry = null; this.#clearSoftRefreshIntentTimer(); this.#clearTerminalIntent(); this.#listeners.clear(); @@ -2157,11 +2712,13 @@ export class DefaultSqlDocumentSession ) { refreshIntent.cancelReason = "disposed"; } - activeCompletion?.ticket?.cancel(); + cancelCompletionTickets(activeCompletion); if (refreshIntent !== activeCompletion) { - refreshIntent?.ticket?.cancel(); + cancelCompletionTickets(refreshIntent); } catalogOwner?.dispose(); + columnOwner?.dispose(); + namespaceOwner?.dispose(); this.#onDispose(); }; } @@ -2170,6 +2727,8 @@ export class DefaultSqlLanguageService implements SqlLanguageService { readonly #catalogCoordinator: SqlCatalogSearchWorkCoordinator | null; + readonly #columnCoordinator: SqlColumnCatalogBatchCoordinator | null; + readonly #namespaceCoordinator: SqlNamespaceCatalogCoordinator | null; readonly #completion: CompletionConfiguration; readonly #dialects: ReadonlyMap; readonly #sessions = new Set>(); @@ -2303,6 +2862,48 @@ export class DefaultSqlLanguageService } this.#catalogCoordinator = coordinator.coordinator; } + + const columns = readOwnDataProperty( + options, + "columns", + "invalid-service-options", + "SQL language service options", + ); + if (!columns.found || columns.value === undefined) { + this.#columnCoordinator = null; + } else { + const coordinator = createSqlColumnCatalogBatchCoordinator({ + provider: columns.value, + }); + if (coordinator.status !== "created") { + throw new SqlSessionError( + "invalid-service-options", + "SQL column catalog provider is invalid", + ); + } + this.#columnCoordinator = coordinator.coordinator; + } + + const namespaces = readOwnDataProperty( + options, + "namespaces", + "invalid-service-options", + "SQL language service options", + ); + if (!namespaces.found || namespaces.value === undefined) { + this.#namespaceCoordinator = null; + } else { + const coordinator = createSqlNamespaceCatalogCoordinator({ + provider: namespaces.value, + }); + if (coordinator.status !== "created") { + throw new SqlSessionError( + "invalid-service-options", + "SQL namespace catalog provider is invalid", + ); + } + this.#namespaceCoordinator = coordinator.coordinator; + } } catch (error) { if (error instanceof SqlSessionError) { throw error; @@ -2365,7 +2966,12 @@ export class DefaultSqlLanguageService const context = cloneContext(candidateContext); resolveDialectRuntime(context, this.#dialects); const catalogContext = resolveCatalogContext(context); - if (catalogContext && !this.#catalogCoordinator) { + if ( + catalogContext && + !this.#catalogCoordinator && + !this.#columnCoordinator && + !this.#namespaceCoordinator + ) { throw new SqlSessionError( "invalid-context", "SQL catalog context requires a configured catalog provider", @@ -2378,6 +2984,8 @@ export class DefaultSqlLanguageService context, this.#dialects, this.#catalogCoordinator, + this.#columnCoordinator, + this.#namespaceCoordinator, this.#completion, () => { this.#sessions.delete(session); @@ -2419,6 +3027,8 @@ export class DefaultSqlLanguageService } this.#sessions.clear(); this.#catalogCoordinator?.dispose(); + this.#columnCoordinator?.dispose(); + this.#namespaceCoordinator?.dispose(); }; } diff --git a/src/vnext/types.ts b/src/vnext/types.ts index 075c403..a0582b4 100644 --- a/src/vnext/types.ts +++ b/src/vnext/types.ts @@ -1,5 +1,15 @@ const revisionBrand: unique symbol = Symbol("SqlRevision"); +export function isDataArray( + value: unknown, +): value is readonly unknown[] { + try { + return Array.isArray(value); + } catch { + return false; + } +} + /** Immutable identity issued by a document session. */ export interface SqlRevision { readonly [revisionBrand]: "SqlRevision"; @@ -135,6 +145,8 @@ export interface OpenSqlDocument { /** Owns all mutable state for one open SQL document. */ export interface SqlDocumentSession { readonly revision: SqlRevision; + /** Invalidates relation, column, and namespace catalog observations. */ + readonly invalidateCatalog: () => SqlRevision; readonly statementBoundaryAt: ( request: SqlStatementBoundaryAtRequest, ) => SqlStatementBoundaryAtResult; @@ -162,10 +174,12 @@ export interface SqlLanguageService { export interface SqlLanguageServiceOptions { readonly catalog?: SqlRelationCatalogProvider | undefined; + readonly columns?: SqlColumnCatalogProvider | undefined; readonly completion?: { readonly catalogResponseBudgetMs?: number | undefined; } | undefined; readonly dialects: readonly SqlDialect[]; + readonly namespaces?: SqlNamespaceCatalogProvider | undefined; } export type SqlSessionErrorCode = @@ -192,6 +206,12 @@ export class SqlSessionError extends Error { this.code = code; } } +import type { + SqlColumnCatalogProvider, +} from "./column-catalog-types.js"; +import type { + SqlNamespaceCatalogProvider, +} from "./namespace-catalog-types.js"; import type { SqlCompletionRequest, SqlCompletionTask, diff --git a/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts b/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts index 33938c2..95dab26 100644 --- a/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts +++ b/test/vnext-types/marimo-relation-completion-codemirror.test-d.ts @@ -3,6 +3,7 @@ import type { EditorView } from "@codemirror/view"; import type { SqlCompletionInfoResolver, SqlCompletionInfoResolverContext, + SqlEditorStatementGutterOptions, } from "../../src/vnext/codemirror/index.js"; import type { SqlCompletionItem } from "../../src/vnext/relation-completion-types.js"; @@ -32,6 +33,14 @@ declare const item: SqlCompletionItem; const resolved = resolveInfo(item, { signal: new AbortController().signal, }); +const gutter: SqlEditorStatementGutterOptions = { + hideWhenNotFocused: true, + showInactive: true, +}; +const invalidGutter: SqlEditorStatementGutterOptions = { + // @ts-expect-error gutter flags are boolean + showInactive: "yes", +}; // @ts-expect-error resolver items are immutable item.label = "changed"; @@ -62,6 +71,8 @@ const resolverWithoutDestroy: SqlCompletionInfoResolver = () => ({ }); void resolved; +void gutter; +void invalidGutter; void resolverReturningNode; void resolverReturningNumber; void resolverReturningReactData; diff --git a/test/vnext-types/marimo-sql-migration.test-d.ts b/test/vnext-types/marimo-sql-migration.test-d.ts new file mode 100644 index 0000000..deebda8 --- /dev/null +++ b/test/vnext-types/marimo-sql-migration.test-d.ts @@ -0,0 +1,591 @@ +import type { + CompletionSource, +} from "@codemirror/autocomplete"; +import type { + ChangeSpec, + EditorState, + StateEffectType, + TransactionSpec, +} from "@codemirror/state"; + +import { + bigQueryDialect, + createSqlLanguageService, + dremioDialect, + duckdbDialect, + postgresDialect, + type SqlCanonicalRelationPath, + type SqlCatalogEpoch, + type SqlCatalogReadyCoverage, + type SqlCatalogRelation, + type SqlCatalogSearchRequest, + type SqlCatalogSearchResponse, + type SqlColumnCatalogBatchRequest, + type SqlColumnCatalogProvider, + type SqlContextInput, + type SqlDocumentContext, + type SqlEmbeddedRegion, + type SqlIdentifierComponent, + type SqlIdentifierPath, + type SqlNamespaceCatalogProvider, + type SqlRelationCatalogProvider, +} from "../../src/vnext/index.js"; +import { + sqlEditor, + type SqlCompletionInfoResolver, + type SqlEditorSupport, +} from "../../src/vnext/codemirror/index.js"; + +type VnextDialectId = + | "bigquery" + | "dremio" + | "duckdb" + | "postgres"; + +type MarimoBackendDialect = + | VnextDialectId + | "postgresql" + | "mysql" + | "oracle" + | "sqlite" + | "snowflake"; + +interface MarimoSqlContext extends SqlDocumentContext { + readonly engine: string; +} + +interface MarimoConnection { + readonly defaultDatabase: string | null; + readonly defaultSchema: string | null; + readonly dialect: MarimoBackendDialect; + readonly engine: string; + /** Changes whenever a same-named backend connection is replaced. */ + readonly incarnation: number; +} + +interface MarimoTableMetadata { + readonly detail: string; + readonly entityId: string; +} + +interface MarimoColumnMetadata { + readonly columnEntityId: string; + readonly detail: string; + readonly relationEntityId: string; +} + +interface MarimoCatalogSnapshot { + readonly epoch: SqlCatalogEpoch; + readonly relations: readonly SqlCatalogRelation[]; +} + +interface MarimoDataTableColumn { + readonly columnEntityId: string; + readonly dataType?: string; + readonly detail?: string; + readonly identifier: SqlIdentifierComponent; + readonly insertText: string; + readonly ordinal: number; +} + +type MarimoDataTableColumnResult = + | { + readonly columns: readonly MarimoDataTableColumn[]; + readonly coverage: "complete" | "partial"; + readonly relationEntityId: string; + readonly requestKey: string; + readonly status: "ready"; + } + | { + readonly requestKey: string; + readonly status: "loading"; + } + | { + readonly code: + | "authentication" + | "authorization" + | "invalid-configuration" + | "rate-limited" + | "unavailable" + | "unknown"; + readonly requestKey: string; + readonly retry: "after-invalidation" | "never" | "next-request"; + readonly status: "failed"; + }; + +interface MarimoDataTableColumnBatch { + readonly epoch: SqlCatalogEpoch; + readonly relations: readonly MarimoDataTableColumnResult[]; +} + +interface MarimoNamespaceProjection { + readonly entityId: string; + readonly kind: "catalog" | "dataset" | "project" | "schema"; + readonly path: SqlIdentifierPath; + readonly scope: string; +} + +declare const catalogByScope: + ReadonlyMap; +declare const tableMetadataById: + ReadonlyMap; +declare const columnMetadataByProvenance: + ReadonlyMap; +declare const namespaceProjectionByScope: + ReadonlyMap; +declare const subscribeToCatalogScope: ( + scope: string, + listener: (epoch: SqlCatalogEpoch) => void, +) => () => void; +declare const searchMarimoCatalogIndex: ( + snapshot: MarimoCatalogSnapshot, + request: SqlCatalogSearchRequest, +) => { + readonly coverage: SqlCatalogReadyCoverage; + readonly relations: readonly SqlCatalogRelation[]; +}; +declare const loadMarimoDataTableColumnBatch: ( + request: SqlColumnCatalogBatchRequest, + signal: AbortSignal, +) => Promise; +declare const variableCompletionSource: CompletionSource; +declare const keywordCompletionSource: CompletionSource; +declare const legacySchemaCompletionSource: CompletionSource; + +interface ReactRootLike { + readonly render: (value: unknown) => void; + readonly unmount: () => void; +} + +declare function createReactRoot(container: Element): ReactRootLike; + +function connectionScope( + connection: MarimoConnection, +): string { + return `${connection.engine}:${connection.incarnation}`; +} + +function identifier(value: string): SqlIdentifierPath[number] { + return { quoted: false, value }; +} + +function searchPaths( + connection: MarimoConnection, +): readonly SqlIdentifierPath[] { + const path: SqlIdentifierPath[number][] = []; + if (connection.defaultDatabase !== null) { + path.push(identifier(connection.defaultDatabase)); + } + if (connection.defaultSchema !== null) { + path.push(identifier(connection.defaultSchema)); + } + return path.length === 0 ? [] : [path]; +} + +function contextFor( + route: VnextCompletionRoute, +): SqlContextInput { + const { connection } = route; + return { + catalog: { + scope: connectionScope(connection), + searchPath: searchPaths(connection), + }, + dialect: route.dialect, + engine: connection.engine, + }; +} + +const marimoCatalogProvider: SqlRelationCatalogProvider = { + id: "marimo-datasets", + search: async ( + request, + signal, + ): Promise => { + signal.throwIfAborted(); + const snapshot = catalogByScope.get(request.scope); + if (snapshot === undefined) { + return { + code: "invalid-configuration", + epoch: { generation: 0, token: "missing-scope" }, + retry: "after-invalidation", + status: "failed", + }; + } + const result = searchMarimoCatalogIndex(snapshot, request); + return { + coverage: result.coverage, + epoch: snapshot.epoch, + relations: result.relations, + status: "ready", + }; + }, + subscribe: (scope, onInvalidation) => { + const unsubscribe = subscribeToCatalogScope(scope, (epoch) => { + onInvalidation({ epoch }); + }); + return (): undefined => { + unsubscribe(); + return undefined; + }; + }, +}; + +const marimoColumnProvider: SqlColumnCatalogProvider = { + id: "marimo-datatable-columns", + loadColumns: async (request, signal) => { + signal.throwIfAborted(); + const batch = await loadMarimoDataTableColumnBatch(request, signal); + signal.throwIfAborted(); + return { + epoch: batch.epoch, + relations: batch.relations.map((result) => { + if (result.status !== "ready") return result; + return { + columns: result.columns.map((column) => ({ + columnEntityId: column.columnEntityId, + identifier: column.identifier, + insertText: column.insertText, + ordinal: column.ordinal, + ...(column.dataType === undefined + ? {} + : { dataType: column.dataType }), + ...(column.detail === undefined + ? {} + : { detail: column.detail }), + })), + coverage: result.coverage, + relationEntityId: result.relationEntityId, + requestKey: result.requestKey, + status: result.status, + }; + }), + }; + }, +}; + +const marimoNamespaceProvider: SqlNamespaceCatalogProvider = { + id: "marimo-namespaces", + search: async (request, signal) => { + signal.throwIfAborted(); + const projections = + namespaceProjectionByScope.get(request.scope) ?? []; + return { + containers: projections.flatMap((projection) => { + const first = projection.path[0]; + if (first === undefined) return []; + const canonicalPath = [ + { + ...first, + role: projection.kind, + }, + ...projection.path.slice(1).map((component) => ({ + ...component, + role: projection.kind, + })), + ] as const; + return [{ + canonicalPath, + containerEntityId: projection.entityId, + insertText: + (canonicalPath[canonicalPath.length - 1] ?? first).value, + matchQuality: "exact" as const, + }]; + }), + coverage: "complete", + epoch: catalogByScope.get(request.scope)?.epoch ?? { + generation: 0, + token: "missing-scope", + }, + status: "ready", + }; + }, +}; + +// One caller-owned service is shared by every SQL editor support/view. +const sharedSqlService = + createSqlLanguageService({ + catalog: marimoCatalogProvider, + columns: marimoColumnProvider, + dialects: [ + bigQueryDialect(), + dremioDialect(), + duckdbDialect(), + postgresDialect(), + ], + namespaces: marimoNamespaceProvider, + }); + +const infoResolver: SqlCompletionInfoResolver = ( + item, + { signal }, +) => { + signal.throwIfAborted(); + const metadata = item.provenance.kind === "catalog" + ? tableMetadataById.get(item.provenance.entityId) + : item.provenance.kind === "column-catalog" + ? columnMetadataByProvenance.get( + `${item.provenance.scope}\0${item.provenance.relationEntityId}\0${item.provenance.columnEntityId}`, + ) + : undefined; + if (metadata === undefined) return null; + const dom = document.createElement("div"); + const root = createReactRoot(dom); + root.render(metadata.detail); + return { + destroy: () => root.unmount(), + dom, + }; +}; + +/** + * Returns the complete, ordered region set for the resulting document. + * Regions include both braces, matching marimo's current `{python}` syntax. + */ +function pythonTemplateRegions( + text: string, +): readonly SqlEmbeddedRegion[] { + const regions: SqlEmbeddedRegion[] = []; + for (let index = 0; index < text.length;) { + if (text[index] !== "{") { + index += 1; + continue; + } + if (text[index + 1] === "{") { + index += 2; + continue; + } + const close = text.indexOf("}", index + 1); + const to = close === -1 ? text.length : close + 1; + if (to > index) { + regions.push({ + from: index, + language: "python", + to, + }); + } + index = Math.max(index + 1, to); + } + return regions; +} + +/** + * Embedded regions are half-open, so an unmatched expression ending at EOF + * also needs an insertion-point gate at `doc.length`. + */ +function sqlCompletionPositionAllowed( + state: EditorState, + position: number, +): boolean { + const prefix = state.sliceDoc(0, position); + let inPython = false; + for (let index = 0; index < prefix.length; index += 1) { + if (inPython) { + if (prefix[index] === "}") inPython = false; + continue; + } + if ( + prefix[index] === "{" && + prefix[index + 1] !== "{" + ) { + inPython = true; + continue; + } + if ( + prefix[index] === "{" && + prefix[index + 1] === "{" + ) { + index += 1; + } + } + return !inPython; +} + +interface VnextCompletionRoute { + readonly connection: MarimoConnection; + readonly dialect: VnextDialectId; + readonly kind: "vnext"; +} + +type CompletionRoute = + | { + readonly kind: "legacy"; + readonly source: CompletionSource; + } + | VnextCompletionRoute; + +function completionRoute( + connection: MarimoConnection, +): CompletionRoute { + switch (connection.dialect) { + case "bigquery": + return { connection, dialect: "bigquery", kind: "vnext" }; + case "dremio": + return { connection, dialect: "dremio", kind: "vnext" }; + case "duckdb": + return { connection, dialect: "duckdb", kind: "vnext" }; + case "postgres": + case "postgresql": + return { connection, dialect: "postgres", kind: "vnext" }; + case "mysql": + case "oracle": + case "sqlite": + case "snowflake": + return { kind: "legacy", source: legacySchemaCompletionSource }; + } +} + +function createVnextSupport( + route: VnextCompletionRoute, + initialText: string, +): SqlEditorSupport { + return sqlEditor({ + autocomplete: { + defaultKeymap: false, + externalSources: [ + variableCompletionSource, + keywordCompletionSource, + ], + infoResolver, + isCompletionPositionAllowed: sqlCompletionPositionAllowed, + }, + initialContext: contextFor(route), + initialEmbeddedRegions: pythonTemplateRegions(initialText), + service: sharedSqlService, + statementGutter: { + hideWhenNotFocused: true, + showInactive: true, + }, + }); +} + +interface MarimoSqlMetadata { + readonly route: VnextCompletionRoute; +} + +declare const setMarimoSqlMetadata: + StateEffectType; +declare const reconfigureSqlDialect: + StateEffectType; + +/** + * Marimo owns this transaction seam. `resultingText` is the document after + * `changes`; all interpretation inputs are emitted in the same transaction. + */ +function atomicSqlInputTransaction( + support: SqlEditorSupport, + input: { + readonly changes?: ChangeSpec; + readonly metadata: MarimoSqlMetadata; + readonly resultingText: string; + }, +): TransactionSpec { + const { route } = input.metadata; + const transaction: TransactionSpec = { + effects: [ + setMarimoSqlMetadata.of(input.metadata), + support.contextEffect.of(contextFor(route)), + reconfigureSqlDialect.of(route.dialect), + support.embeddedRegionsEffect.of( + pythonTemplateRegions(input.resultingText), + ), + ], + }; + return input.changes === undefined + ? transaction + : { ...transaction, changes: input.changes }; +} + +declare const duckdbRoute: VnextCompletionRoute; +declare const secondDuckdbRoute: VnextCompletionRoute; +declare const mysqlConnection: MarimoConnection & { + readonly dialect: "mysql"; +}; + +const firstSupport = createVnextSupport( + duckdbRoute, + "SELECT * FROM {df}", +); +const secondSupport = createVnextSupport( + secondDuckdbRoute, + "SELECT * FROM users", +); +const atomicSwitch = atomicSqlInputTransaction(firstSupport, { + changes: { from: 14, insert: "orders", to: 18 }, + metadata: { route: secondDuckdbRoute }, + resultingText: "SELECT * FROM orders", +}); +const legacyRoute = completionRoute(mysqlConnection); +if (legacyRoute.kind === "legacy") { + const source: CompletionSource = legacyRoute.source; + void source; +} + +const relationPath = [ + { quoted: false, role: "catalog", value: "memory" }, + { quoted: false, role: "schema", value: "main" }, + { quoted: false, role: "relation", value: "users" }, +] satisfies SqlCanonicalRelationPath; + +const coldColumnBatch = { + dialectId: "duckdb", + expectedEpoch: null, + relations: [ + { + path: [ + { quoted: false, value: "main" }, + { quoted: false, value: "users" }, + ], + requestKey: "binding:users", + }, + { + path: [ + { quoted: false, value: "main" }, + { quoted: false, value: "orders" }, + ], + requestKey: "binding:orders", + }, + ], + searchPaths: [[{ quoted: false, value: "main" }]], + scope: "duckdb:42", +} satisfies SqlColumnCatalogBatchRequest; + +const columnProviderStateExamples = [ + { + columns: [{ + columnEntityId: "connection:users:customer-id", + dataType: "VARCHAR", + detail: "Customer identifier", + identifier: { quoted: true, value: "customer id" }, + insertText: '"customer id"', + ordinal: 0, + }], + coverage: "partial", + relationEntityId: "connection:users", + requestKey: "binding:users", + status: "ready", + }, + { + requestKey: "binding:orders", + status: "loading", + }, + { + code: "authorization", + requestKey: "binding:private-orders", + retry: "after-invalidation", + status: "failed", + }, +] satisfies readonly MarimoDataTableColumnResult[]; + +void atomicSwitch; +void coldColumnBatch; +void columnProviderStateExamples; +void firstSupport; +void marimoCatalogProvider; +void marimoColumnProvider; +void namespaceProjectionByScope; +void relationPath; +void secondSupport; + +// The application, not either editor support, owns the shared service. +sharedSqlService.dispose();