From d7470d763552d61f4146d70af1906bed3959ce4a Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 15:01:06 -0700 Subject: [PATCH 01/14] Define scope-authorized current-head exports --- docs/cloud-sandbox-data-plane.md | 460 +++++++++++++++++++++---------- 1 file changed, 315 insertions(+), 145 deletions(-) diff --git a/docs/cloud-sandbox-data-plane.md b/docs/cloud-sandbox-data-plane.md index ec43e0f8..ea3cd6e1 100644 --- a/docs/cloud-sandbox-data-plane.md +++ b/docs/cloud-sandbox-data-plane.md @@ -1,8 +1,8 @@ # Cloud Sandbox Data Plane Architecture -Status: proposed target architecture with a pragmatic v1 delivery path +Status: approved target architecture with a pragmatic v1 delivery path -Date: 2026-07-17 +Date: 2026-07-23 Scope: Locality backend, sandbox client, connector execution, replica transfer, permissions, mutations, search, security, and migration @@ -40,11 +40,13 @@ The main decisions are: agent sandbox. The existing direct connector mode remains available to the local-first desktop and headless `loc` client; a mount uses exactly one mode at a time. -4. Resolve authorization and job filters once in an indexed PostgreSQL query - against a complete ready revision. Stream exactly the selected ordinary files - as a transient standard tar response with negotiated Zstandard wire - compression; do not build, compact, authorize, or retain profile-specific - packs or client-side catalogs. +4. Authorize sessions with stable provider scopes rather than enumerated file + IDs. For each export attempt, revalidate current grants, freshness, filters, + and quotas, resolve the current complete published generation for each source + in one PostgreSQL snapshot, and stream exactly the selected ordinary files as + a transient standard tar response with negotiated Zstandard wire compression. + Do not build, compact, authorize, or retain profile-specific packs or + client-side catalogs. 5. Materialize selected content as ordinary files on task-local disk before the agent starts. `rg`, `grep`, compilers, editors, and ordinary POSIX calls then operate at local filesystem speed with no read-time network calls. @@ -82,9 +84,10 @@ The main decisions are: isolation limits prevent the service from saturating supported sandbox links. ClickHouse is not a v1 dependency or mutation authority. 13. Make the three high-volume PostgreSQL serving tables locality-aware from the - beginning: identical small tenant/source hash partitions, monotonic - source/root export and content-storage IDs, a minimal baseline index set, and - ordered backfill writes. Do not depend on periodic heap clustering; tune + beginning: identical small tenant/source hash partitions, stable scope IDs, + canonical bytewise logical-path ordering, monotonic content-storage IDs, a + minimal baseline index set, and ordered backfill writes. Do not depend on a + globally allocated per-file export rank or periodic heap clustering; tune partition count, secondary indexes, and physical rewrites only from measured usage. 14. Deploy the first managed cell on AWS as one container image in separate ECS @@ -114,9 +117,10 @@ a single-tenant or Notion-only pilot. It includes: - coarse administrator-selected scopes such as roots, shared drives, channels, mailboxes, teams, or projects, plus explicit broad pilot groups/grants; - exact server-side authorization/profile filtering and one negotiated - Zstd-or-identity streaming file export per session; + Zstd-or-identity streaming file export per attempt; - ordinary task-local files, including read-only and explicitly writable paths; -- replica-at-start freshness with no live sandbox delta stream; +- current-complete-head selection per export attempt with no live in-place + sandbox delta stream; a refresh stages and atomically publishes a new tree; - no read-only metadata import; a small SQLite session store tracks only writable entries, local changes, journals, and submitted changesets; - PostgreSQL title/path/content search for profile discovery, with local `rg` as @@ -460,6 +464,28 @@ Provider identities and groups link to Locality IDs through stable provider subject IDs. Display names are never identity, and email addresses are used for linking only after issuer/domain verification. +### Stable Source Scopes + +Authorization selectors name stable provider containers and their documented +membership semantics, not the files currently beneath them: + +- Notion root page or database ID, including descendants reached by connector + traversal; +- Slack channel/conversation ID, including messages, threads, replies, and files + owned by those messages; +- Granola collection, team, or folder ID, or an explicitly selected meeting ID; +- Gmail mailbox plus stable label ID, with an explicit thread ID only when it is + intentionally selected. Arbitrary search text is a narrowing filter, not an + authorization scope; +- Drive shared-drive or folder ID, including descendants; and +- GitHub organization or repository ID. + +Connector traversal/provider facts write membership at ingestion. A title, +projected path, teamspace label, or search result cannot manufacture authority. +Attachments inherit the memberships of their owning resource. Adding or removing +a descendant changes the next published generation and therefore the next export +attempt; it does not require rewriting a grant/session for every file. + ### 1. Source Resource A source resource is a provider-owned identity: @@ -513,13 +539,17 @@ connector-supported action set. Effective actions are session policy, not intrinsic content metadata. The many-to-many `projection_inputs` relationship lets connectors aggregate or split resources without corrupting remote identity. -The backend also assigns physical-serving hints: a stable `scope_root_id`, a -monotonic `export_order` within that source/root, and a monotonic internal +The backend also assigns a stable `scope_id` and a monotonic internal `content_storage_id` for each content version. These are not provider identity -or wire-format fields. They let ingestion place related rows together and let -exports read bodies in storage-friendly order without changing filesystem paths. -Allocate ID ranges through ordinary connection-scoped database state; do not -create a PostgreSQL sequence or partition for every tenant, source, or root. +or wire-format fields. `scope_id` records an ingestion-derived provider +membership such as a Notion root, Slack channel, Gmail label, Drive folder, or +GitHub repository. Titles, projected paths, and search results never establish +membership. The common single-membership case stores the scope directly on the +projection row; connectors with genuine many-to-many membership use an indexed +bridge. Ingestion writes related rows in canonical logical-path order where +practical, but correctness and wire order never depend on a mutable allocated +`export_order`. Do not create a PostgreSQL sequence or partition for every +tenant, source, or scope. Its `LogicalPath` is a normalized portable relative path, not a Rust `PathBuf` or an OS-specific absolute path. The same `ProjectionEntry` can therefore be @@ -529,37 +559,57 @@ path, hydration/materialization state, dirty state, and local-store identity. The current `TreeEntry` can remain as a compatibility adapter while connectors migrate; it should not become the backend's canonical source object. -### 4. Replica Revision And Session Export +### 4. Published Generation And Session Export -A replica revision is a durable point-in-time publication for one source -connection. It records the source watermark, projection revision, coverage, +A published generation is a durable point-in-time publication for one source +connection. It records the source watermark, projection sequence, coverage, freshness observation, and publication state. Projection content uses immutable -version rows plus current/version-range mappings, so publishing a revision does -not duplicate a physical corpus. +version rows plus current/version-range mappings, so publishing a generation +does not duplicate a physical corpus. Its immutable manifest retains the exact +scope memberships, logical paths, connector/source-supported action ceilings, +content references, + hashes, sizes, connector/source-supported action ceilings, and exact attachment + object versions required for retries and audit after a newer head publishes. ```text provider watermark -> immutable source/content versions - -> transactionally complete ready projection revision + -> transactionally complete ready published generation -sandbox session + current grants + profile/runtime filters +sandbox session + stable source scopes + -> export-attempt revalidation + current complete generation vector -> one exact PostgreSQL selection query -> one transient Zstd-compressed tar stream ``` The session row records the tenant, principal/workload, profile and policy -revisions, source watermarks, selected replica revisions, query/filter digest, -expiry, and delivered file/byte counts. The exporter runs under the session's +revisions, ordered stable source scopes, requested actions, query/filter digest, +limits, and expiry. It does not enumerate projection IDs or pin source +generations. On the first request for an export attempt, the exporter rechecks +current provider visibility, grants, scope activity, freshness, filters, and +quotas in one PostgreSQL MVCC snapshot, then seals an ordered +`(source_connection_id, generation_id)` vector and exact offer to the attempt. +A retry of that attempt reuses the vector and offer; a new attempt on the same +session may select newer complete heads. The exporter runs under the session's tenant and policy context and never accepts arbitrary SQL from the client. +Overlapping scopes emit a projection once. Each membership independently +intersects the provider ceiling, tenant grant, acting-principal grant, workload +profile, session actions, and runtime filters. The lowest configured scope +ordinal that alone authorizes every required action wins. Partial actions from +different scopes are never combined. Distinct projections resolving to the same +logical path remain a hard error. + Two sets remain distinct: - **authorized/selected set**: entries matching current `DataGrant`, provider ACL facts, profile roots, and runtime filters; and - **delivered set**: entries the exporter actually wrote to the response. -In v1 these sets must be equal. There is no authorized overfetch or client-side -confidentiality filtering. A trusted host-side bootstrapper may report which +For a successfully completed attempt these sets must be equal. An aborted, +cancelled, or disconnected attempt records an exact terminal state and may have +delivered fewer bytes, but it never publishes the partial local tree. There is +no authorized overfetch or client-side confidentiality filtering. A trusted host-side bootstrapper may report which entries were materialized, but that telemetry is not an authorization control. The complete delivered inventory may be retained as compact audit rows or a database-generated digest according to policy; it is not a reusable data @@ -593,7 +643,7 @@ Read-only and writable content deliberately have different local costs: | Content class | Local state | Update behavior | | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------- | -| Read-only | One materialized file. The backend session/audit state retains its revision and delivered identity; no local shadow or metadata row is required. | Fixed for an ephemeral session. A later session receives a newer replica revision; no merge or conflict state exists. | +| Read-only | One materialized file. The backend attempt/audit state retains its generation vector and delivered identity; no local shadow or metadata row is required. | Fixed for one published local tree. A later refresh attempt may stage a newer complete generation and atomically replace the tree; no merge or conflict state exists. | | Writable but clean | Materialized working file plus a compressed/reflinked baseline and provider precondition. No local copy of remote head. | Eligible for local diff and explicit push. | | Writable and dirty | Working bytes, retained baseline, dirty journal, and changeset state for affected entries. | Three-way comparison occurs only when provider head differs at submit/apply. | | Conflicted | Base, working, and fetched remote content for affected entries only. | Kept until rebase, resolution, rejection, or retention expiry. | @@ -626,8 +676,8 @@ first physical schema: | Source state | `source_resources`, `source_edges`, `source_versions`, `resource_acl_facts`, `access_sets`, `access_descriptors`, `ingest_events`, `tombstones` | | Content | `content_versions`, `projection_entries`, `projection_entry_versions`, `projection_inputs`, `large_attachments` | | Policy | `data_grants`, `data_grant_revisions`, `workspace_profiles`, `workspace_profile_revisions`, `compiled_policy_bindings` | -| Replica | `replica_revisions`, `replica_source_watermarks` | -| Runtime | `sandbox_sessions`, `session_capabilities`, `session_deliveries`, `materialization_receipts`, `client_checkpoints`, `write_claims`, `jobs` | +| Replica | `replica_revisions`, `replica_source_watermarks`, `source_scopes`, immutable generation manifests and scope memberships | +| Runtime | `sandbox_sessions`, `sandbox_session_scopes`, export attempts/offers, `session_deliveries`, `materialization_receipts`, `client_checkpoints`, `write_claims`, `jobs` | | Mutation | `changesets`, `change_operations`, `approvals`, `apply_attempts`, `apply_effects`, `reconciliation_results` | | Governance | `audit_events`, `retention_holds`, `deletion_requests`, `export_checkpoints` | @@ -665,10 +715,10 @@ materializing every logical subtype: | `source_checkpoints` | Durable cursor/fingerprint and completeness per root, channel, drive, mailbox, or other resumable connector scope. | | `source_resources`, `source_edges`, `source_versions` | Preserve stable provider identity, hierarchy/membership, immutable versions, current pointer, provider precondition, tombstone, and ACL observations. | | `access_sets`, `access_set_subjects` | Normalizes equivalent observed reader sets so indexed authorization joins do not evaluate provider ACL JSON per file. | -| `content_versions`, `projection_entries`, `projection_entry_versions`, `projection_inputs` | Separates immutable bodies from narrow current/versioned selection rows; carries source/root/export/storage locality keys; and preserves the many-to-many source-to-filesystem mapping. | +| `content_versions`, `projection_entries`, `projection_entry_versions`, `projection_inputs` | Separates immutable bodies from narrow current/versioned selection rows; carries source/scope/path/storage locality keys; and preserves the many-to-many source-to-filesystem mapping. | | `policies`, `policy_revisions` | Stores both `DataGrant` and `WorkspaceProfile` documents by kind; compiled results are rebuildable cache fields or objects. | | `replica_revisions` | Records source watermark, coverage, freshness observation, projection sequence, ready state, and PostgreSQL commit/replay position. | -| `sandbox_sessions`, `session_deliveries` | Includes capability hash/claims, pinned revisions, filter/query digest, selected/delivered counts and bytes, plus an optional exact delivered inventory when policy requires it. | +| `sandbox_sessions`, export attempts, `session_deliveries` | Sessions include capability hash/claims, ordered stable scopes, actions, limits, and filter/query digest. Attempts bind an immutable generation vector and exact offer/receipt, selected/delivered counts and bytes, plus an optional exact delivered inventory when policy requires it. | | `jobs` | Durable leased work, provider-event dedupe, retry, cooldown, and idempotency state. | | `search_documents` | Rebuildable PostgreSQL full-text projection. | | `changesets`, `change_operations` | Operations hold attempt, effect, reconciliation, and error JSON until scale or retention justifies separate tables. | @@ -679,9 +729,11 @@ the boundaries that are expensive to retrofit: tenant-leading keys and RLS, stable provider IDs, immutable versions/provider preconditions, resource versus projection identity, projection inputs, access-set IDs, policy/replica/format revisions, resumable checkpoint identity, idempotency keys, and audit correlation -IDs. Also bake `source_connection_id`, `scope_root_id`, `export_order`, and -`content_storage_id` into the serving schema because adding physical locality -after a large corpus exists is expensive. It is safe to normalize typed +IDs. Also bake `source_connection_id`, stable `scope_id`, canonical +`logical_path`, and `content_storage_id` into the serving schema because adding +membership and physical locality after a large corpus exists is expensive. Do +not make a globally allocated export rank canonical: insertions, moves, and +renames must not require renumbering unrelated entries. It is safe to normalize typed principal subtypes, webhook subscriptions, compiled policy bindings, receipts, approvals, apply attempts/effects, holds, or exports later through ordinary migrations. @@ -810,10 +862,12 @@ The managed-cell contract is: service roles to their cell/tenant responsibilities, and audit database, object, and KMS access; - issue only a short-lived Locality session capability to the sandbox. The fixed - export endpoint binds it to tenant, workload, policy/profile revisions, source - revisions, and expiry. Session creation accepts only declarative values - validated against the published profile; export accepts no new filters and - neither endpoint accepts arbitrary SQL; + export endpoint binds it to tenant, workload, policy/profile revisions, stable + source scopes, actions, narrowing filters/limits, and expiry. The export + attempt binds the current authorized generation vector and exact offer. + Session creation accepts only declarative values validated against the + published profile; export accepts no new filters and neither endpoint accepts + arbitrary SQL; - atomically materialize into a staging directory and expose the tree only after the tar response completes. A truncated response is discarded and restarted; resumable cursors are added only if measurements justify them; and @@ -1064,7 +1118,7 @@ The read path separates narrow selection rows from immutable bodies: ```text projection_entries - stable projection identity, current pointer, source/root/export order + stable projection identity, current pointer, source/scope/canonical path projection_entry_versions source/revision validity, logical path, root, kind, ACL set, @@ -1084,16 +1138,19 @@ materialized columns. Connector-specific uncommon fields may remain JSONB, but v1 does not evaluate arbitrary user-supplied JSONPath or SQL. Each projection-version row has a validity interval or equivalent publication -sequence. Publishing a new revision inserts only changed content/projection rows -and closes or supersedes affected prior rows. A query for a pinned revision sees -exactly the entries valid at that revision; it does not copy an entire physical -snapshot. - -The ordinary fast path reads the narrow current-head fields when the pinned -revision is still the source's current ready revision. If publication advanced -between session creation and export, the same repository falls back to the -version-validity rows. Both paths run in the export's repeatable-read snapshot -and must return identical logical rows for the same revision. +sequence and immutable generation membership facts. Publishing a new generation +inserts only changed content/projection rows and closes or supersedes affected +prior rows. Current serving rows accelerate a new attempt against the published +head; immutable version rows reconstruct an older attempt-bound generation +without copying rows into the session. + +The fast path reads narrow current-head rows after joining the session's small +ordered scope set. The export-attempt transaction captures a complete authorized +metadata manifest containing immutable inline-content or exact S3-version +references, then closes before body streaming. A retry of the same attempt reads +the same generation vector and manifest even after publication advances or the +API process restarts. Active attempts and the retry/receipt retention horizon +fence generation and attachment garbage collection. A session export is logically one fixed query: @@ -1103,15 +1160,19 @@ WITH selected AS MATERIALIZED ( SELECT e.tenant_id, e.source_connection_id, e.projection_id, - e.scope_root_id, - e.export_order, + min_authorizing_scope_ordinal(e) AS winning_scope_ordinal, e.logical_path, + logical_parent(e.logical_path) AS parent_path, e.file_kind, e.effective_actions, e.provider_version, e.content_storage_id, e.large_attachment_id - FROM authorized_projection_entries(:session_id, :replica_revisions) AS e + FROM authorized_current_projection_entries( + :session_id, + :export_attempt_id, + :source_generation_vector + ) AS e WHERE profile_predicates_match(e, :validated_runtime_values) ) SELECT e.logical_path, @@ -1127,10 +1188,10 @@ LEFT JOIN content_versions AS c ON c.tenant_id = e.tenant_id AND c.source_connection_id = e.source_connection_id AND c.content_storage_id = e.content_storage_id -ORDER BY e.source_connection_id, - e.scope_root_id, - e.export_order, - e.content_storage_id; +ORDER BY e.winning_scope_ordinal, + e.parent_path COLLATE "C", + e.logical_path COLLATE "C", + e.projection_id; ``` The real implementation expresses supported predicates as parameterized SQL and @@ -1146,12 +1207,13 @@ file. Rejected metadata rows never fetch their content body. The baseline physical indexes are deliberately small: -- current projection selection on `(tenant_id, source_connection_id, -scope_root_id, export_order)` including only the narrow ACL, filter, - current-version, and `content_storage_id` fields required before body fetch; -- historical revision lookup on `(tenant_id, source_connection_id, -scope_root_id, projection_id, valid_from_sequence DESC)` with the closing - sequence available for filtering; +- current projection selection on `(tenant_id, source_connection_id, scope_id, + logical_path COLLATE "C", projection_id)` including only the narrow ACL, + filter, current-version, and content-reference fields required before body + fetch; +- historical generation lookup on `(tenant_id, source_connection_id, scope_id, + projection_id, valid_from_sequence DESC)` with the closing sequence and + canonical logical path available for filtering/order; - authorization lookup on `(tenant_id, subject_id, access_set_id)`; and - immutable body lookup on `(tenant_id, source_connection_id, content_storage_id)`. @@ -1159,10 +1221,13 @@ content_storage_id)`. Do not add one broad covering index containing paths, bodies, and every optional filter. Add a time/kind/provider-specific index only after real query plans show that it avoids enough heap work to repay its write, WAL, vacuum, and cache cost. -Use a custom plan for each export query shape because a broad root and a narrow -filter can require very different plans. Session creation uses statistics for -estimates; it does not run an exact full `COUNT(*)` pass before streaming. Hard -entry/byte limits are enforced as rows are produced. +Use a custom plan for each export query shape because a broad scope and a narrow +filter can require very different plans. The index accelerates authorization and +canonical file-path access but need not provide the entire archive order. A +bounded metadata sort is acceptable until real plans justify another narrow +index. Session creation uses statistics for estimates; export-attempt preflight +performs the exact metadata selection needed to seal the offer and quotas before +any body or S3 read. ### Revision Publication And Source Freshness @@ -1177,11 +1242,12 @@ provider hint/change cursor/scheduled poll -> complete ready replica revision ``` -Work may span idempotent jobs, but the final publisher transaction advances a -revision to `ready` only after all required resource, edge, ACL, content, -projection, checkpoint, and search-input writes for its declared coverage have -completed. A session sees the old complete revision or the new complete -revision, never a partially rendered tree. +Work may span idempotent jobs, but the final lease-fenced publisher transaction +advances the current head to `ready` only after all required resource, edge, +ACL, content, projection, scope-membership, checkpoint, and search-input writes +for its declared coverage have completed. An export attempt sees the old +complete generation or the new complete generation, never a partially rendered +tree. A stale or expired worker cannot overwrite a newer published head. Each source revision records: @@ -1202,8 +1268,9 @@ freshness: waitTimeout: 30s ``` -Session creation starts immediately when selected scopes are complete, have no -known unapplied event, and meet `maxAge`. A known pending event or stale +Session creation may report readiness when selected scopes are complete, have no +known unapplied event, and meet `maxAge`; export-attempt preflight rechecks those +facts before sealing a generation vector. A known pending event or stale observation is promoted ahead of backfill; the request waits only up to its bounded timeout. If freshness cannot be established, the API returns `source_bootstrapping`, `source_stale`, or `source_unavailable` with progress @@ -1218,9 +1285,16 @@ regardless of read freshness mode. ### Consistent Streaming Export -After authorizing a session and resolving the newest acceptable ready revision -for each source, the exporter opens a read-only `REPEATABLE READ` transaction. -The revision lookup and row stream share that database snapshot. Updates that +For the first request of an export attempt, the exporter opens one writable +`REPEATABLE READ` transaction, rechecks current authorization/freshness/limits, +resolves the newest acceptable ready generation for every source, and seals the +ordered source/generation vector plus exact selection offer. "Exact" means the +selected manifest, counts, content bytes, and inventory digest are fixed; it +does not require a speculative first tar/Zstd pass. The same snapshot captures +the complete immutable metadata manifest and atomically persists the attempt, +generation vector, exact selection offer, and retention references before +commit. The transaction can then close while +the exporter fetches bodies through captured immutable references. Updates that commit while the stream is running are not mixed into the response. The client advertises `Accept-Encoding: zstd, identity`. The preferred v1 @@ -1234,8 +1308,23 @@ Content-Encoding: zstd `identity` remains a required fallback and benchmark control. In either mode: - the tar is generated on demand and is never stored as a canonical artifact; -- each selected projection becomes one ordinary tar member with its logical path, - size, and non-executable mode; +- the sealed v2 offer distinguishes one control member, file count, directory + count, total archive entry count, selected content bytes, and a canonical + inventory digest so + parent directory headers cannot create an ambiguous count. It does not + pre-render tar/Zstd merely to predict encoded length or stream digest; the + completion receipt records actual decoded/encoded bytes and stream digest + after the one streaming pass; +- the reserved writable-metadata control member is archive record class 0 and + is always first, even when its writable-entry list is empty. It participates + in total archive-entry count and the canonical inventory digest; +- directory headers are record class 1 and follow in + `(depth, logical_path COLLATE "C")` + order, so every unique parent precedes descendants; +- each selected projection is record class 2 and becomes one ordinary tar member with its logical + path, size, and non-executable mode. File members follow + `(winning_scope_ordinal, parent_path COLLATE "C", logical_path COLLATE "C", + projection_id)` order; - a reserved first member contains versioned session metadata only for writable paths: projection/resource identity, delivered content hash, provider precondition, effective actions, and baseline requirements. The client consumes @@ -1246,7 +1335,20 @@ Content-Encoding: zstd - the client validates and extracts into a staging directory with path, link, device, case, Unicode, entry-count, byte, and disk limits, then atomically publishes the tree only after the HTTP response, Zstd frame, tar stream, and - file writes terminate cleanly. + file writes terminate cleanly and its recomputed canonical inventory digest + equals the sealed offer. + +The v2 canonical inventory digest is SHA-256 over a domain-separated, +length-framed binary preimage. It begins with +`locality.export.inventory.v2\0`; every scalar is an unsigned 64-bit big-endian +length followed by its UTF-8 or raw bytes, and every list begins with an unsigned +64-bit big-endian count. Records follow the archive's class/order above. The +control record includes its path plus exact body SHA-256 and length; a directory +includes its normalized `LogicalPath`; a file includes global winning scope +ordinal, source connection ID, projection ID, normalized logical path, file kind, +sorted effective-action labels, content SHA-256, and byte length. Public exact +goldens bind the preimage and digest so the backend and client cannot disagree +about JSON formatting, collation, or field omission. Start with Zstd level 1. It normally reduces Locality's Markdown/JSON-heavy wire bytes substantially while retaining fast, bounded-memory streaming decode. @@ -1271,11 +1373,13 @@ farm, or another manifest merely to claim parallel decompression. Add such framing only if the decoder is the measured bottleneck rather than PostgreSQL, the network, or filesystem entry creation. -An interrupted v1 export is discarded and restarted against a newly authorized -session snapshot. Add a logical resume cursor only when measured corpus size or +An interrupted v1 export is discarded locally. Retrying the same export attempt +rechecks revocation and replays its already sealed generation vector and exact +offer; starting a new attempt rechecks all authority/freshness/quotas and may +select newer heads. Add a logical byte cursor only when measured corpus size or failure rate justifies its state and consistency complexity. The session -capability is short-lived and revocable; every retry rechecks current policy and -freshness. +capability is short-lived and revocable. Revocation blocks both new attempts and +attempt retries before body/S3 access; it cannot recall bytes already delivered. The backend records source/profile/policy revisions, query digest, selected and delivered file/byte counts, status, duration, and failure. A customer policy may @@ -1296,17 +1400,18 @@ sequenceDiagram O->>L: one-time bootstrap token with profile/runtime scope already sealed L->>A: create session - A->>P: authorize identity, grants, profile, freshness - P-->>A: acceptable ready revisions + estimates - A-->>L: short-lived session + revision/freshness receipt - L->>A: GET session file stream - A->>P: REPEATABLE READ authorized selection + COPY/row stream + A->>P: authorize identity, grants, profile, stable scopes + P-->>A: readiness/freshness + estimates + A-->>L: short-lived scope-authorized session + L->>A: create/open export attempt + A->>P: REPEATABLE READ revalidation + current heads + exact metadata + P-->>A: sealed generation vector + offer A->>B: fetch selected large attachments only A-->>L: transient Zstd(tar) stream L->>F: concurrent decode + safe staged extraction L->>F: writable metadata to SQLite L->>F: atomic publish - L-->>O: ready(path, revisions, file/byte counts) + L-->>O: ready(path, generation vector, file/byte counts) O->>F: start agent ``` @@ -1322,12 +1427,14 @@ than supported sandbox ingress at expected launch concurrency. Optimize the simple path before adding another data store: - keep authorization/filter columns narrow and separately indexed from bodies; -- prune the three serving tables by tenant/source, select through the narrow - current-head or historical-revision index, and fetch bodies in - `(scope_root_id, export_order, content_storage_id)` order; +- prune the three serving tables by tenant/source, join the session's small + ordered scope set, deduplicate overlapping memberships, select through the + narrow current-head or historical-generation index, and fetch bodies in + bounded canonical path batches through immutable content references; - join immutable bodies only after selection and use a bounded streaming transaction with backpressure; -- assign locality keys and insert initial backfill batches in export order. New +- assign locality keys and insert initial backfill batches in canonical + scope/logical-path order. New versions append; correctness never depends on physical heap position; - use LZ4 TOAST compression for new text/JSON/bytea body values when the managed PostgreSQL build supports it. This storage compression is transparent and @@ -1351,10 +1458,13 @@ correctness or publication. Monitor heap/index block reads, index bloat, and specific serving table crosses a measured degradation threshold. A PostgreSQL read replica is the first scale-out step when it provides sufficient -isolation. The session API routes an export to a replica only after its replay -LSN reaches both the selected revision/policy state and the session-creation +isolation. `POST .../export-attempts` always runs and commits its writable atomic +selection/sealing transaction on the primary. Only the subsequent immutable +stream may use a replica, and only after its replay LSN reaches the policy/scope +state, session commit, attempt commit, and every selected published-generation commit; otherwise it waits briefly or uses the primary according to cell policy. -Replica lag can never silently select an older revision or miss the session row. +Replica lag can never silently select an older head or miss the session/attempt +row. ### Deferred ClickHouse Serving Replica @@ -1378,7 +1488,7 @@ sensitive-data backup/retention plane, or read-time deduplication before the benchmark gate is met. A disposable cached export may later accelerate a repeatedly identical -`(tenant, policy revision, profile revision, source revisions, filter digest)` +`(tenant, policy revision, profile revision, source generation vector, filter digest)` query. It is a TTL cache, not canonical state or an ACL composition primitive; cache misses always fall back to the database exporter and correctness never depends on cache compaction. @@ -1437,7 +1547,7 @@ because that would create two independent mutation authorities. The materialized ordinary files are the local read representation. A small writable SQLite session store contains only: -- mount/session identity and delivered source revisions; +- mount/session identity and delivered export-attempt source generations; - write-enabled paths, projection/resource IDs, provider preconditions, and compact effective actions; - per-writable-file baseline references/hashes; @@ -1485,16 +1595,20 @@ as `0555` or `0755`, and all data as non-executable. These modes improve agent behavior but are not the security boundary: an agent running as the owner may change modes. The backend re-authorizes every changeset operation. -### Session Freshness, Not Live Sandbox Deltas +### Current-Head Refresh, Not Live In-Place Sandbox Deltas -An ephemeral sandbox uses immutable source replica revisions selected at session start. V1 -does not subscribe through SSE/WebSocket, poll for content deltas, or rewrite -files while an agent is working. This removes daemon complexity and makes every -local diff relative to an unambiguous delivered base. +An ephemeral sandbox session authorizes stable source scopes. Each export +attempt selects the current complete published generation for every source and +materializes one immutable local tree. V1 does not subscribe through +SSE/WebSocket, poll for content deltas, or rewrite files while an agent is using +that tree. A requested refresh builds a new staging tree and atomically replaces +the old read-only tree only after full validation. This removes daemon +complexity and makes every local diff relative to an unambiguous delivered base. -Correctness at write time does not depend on those revisions still being current. -The backend reads current provider state and checks the delivered precondition -before applying a changeset. Drift produces a no-op, rebase, or conflict. +Correctness at write time does not depend on the delivered generations still +being current. The backend reads current provider state and checks the delivered +precondition before applying a changeset. Drift produces a no-op, rebase, or +conflict. Desktop Live Mode and unusually long-running sandboxes may later use revision deltas. That is a separate capability negotiated by the client; it is not part @@ -1523,7 +1637,7 @@ Where a host-side sidecar is unavailable, the sandbox may hold a short-lived Locality capability. It is deliberately less powerful than a provider token: - bound to tenant, session, profile revision, and action set; -- open its exact export and/or propose changes only; +- create/retry its scope-authorized export attempts and/or propose changes only; - no policy administration or arbitrary search; - short expiry and revocable session ID; and - current policy is rechecked on every submission. @@ -1764,6 +1878,17 @@ unverified email matching never confer access. ### Policy Changes +Each tenant has one monotonic active authorization epoch. Grant, group, +principal-disablement, provider-credential-ceiling, source-scope activation/ +revocation, or provider ACL/access-set observation changes publish their +immutable facts and advance that epoch in one transaction. Sessions retain the +epoch they were issued under for audit, but +export-attempt creation and retry require it to equal the tenant's active head +and re-evaluate the selected scopes. This coarse v1 invalidation may revoke more +sessions than strictly necessary, but it cannot leave a stale grant usable and +does not require rewriting content. A later dependency-aware invalidation index +may reduce blast radius without changing the session/scope protocol. + Changing profile selection creates a new profile revision and changes only the next parameterized export query. Changing a `DataGrant`, group membership, or native ACL updates indexed policy/access facts and invalidates affected session @@ -2020,10 +2145,11 @@ source-controlled namespaces. - Default deny at control plane, database RLS, repository, search, export session, and apply layers. -- A short-lived session capability opens only the fixed export compiled from its +- A short-lived session capability opens only export attempts compiled from its tenant, workload/acting principal, current grants, immutable profile revision, - validated runtime filters, accepted source revisions, and quotas. It cannot - submit SQL or broaden any predicate. + ordered stable source scopes, validated runtime filters, actions, and quotas. + The attempt binds the current complete source generations and exact offer. It + cannot submit SQL or broaden any predicate. - The database query returns only exact authorized/selected rows. The client is never trusted to discard unauthorized paths, metadata, or plaintext. - Policy uses stable source/projection IDs. Paths are display and selection @@ -2057,7 +2183,7 @@ source-controlled namespaces. | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Cross-tenant query or content access | Tenant cell routing, default-deny RLS, tenant-leading keys/predicates, non-owner application role, fixed parameterized export repositories, automated isolation tests. | | Prompt injection and exfiltration | No provider credentials in sandbox, deny-by-default egress, source content never becomes instructions, explicit bounded changesets, destructive-plan denial, narrow session capability. | -| Stolen sandbox capability | Short TTL, exact profile/filter/revision binding, one-session quotas, revocation, current-policy/freshness checks, no provider/database/cloud credential, and audit. | +| Stolen sandbox capability | Short TTL, exact profile/filter/scope binding, attempt-bound generation/offer, one-session quotas, revocation, current-policy/freshness checks, no provider/database/cloud credential, and audit. | | Database credential compromise | Private networking, TLS, managed encryption, narrowly scoped non-owner roles, RLS, secret rotation, no standing human data-reader role, query/access audit, and time-bound break-glass. | | Object-store credential compromise | Large attachments/explicit exports only, SSE-KMS, separate object/KMS roles, opaque tenant-scoped keys, no sandbox credentials, provider audit events. | | Malicious stream/path | Backend-generated tar only, compressed/wire and decoded-byte limits, bounded Zstd window/buffers, safe `openat` extraction, no links/devices, path/collision/reserved-name validation, staged atomic publish. | @@ -2108,8 +2234,9 @@ Every backend-visible action emits an append-only audit event with: - denial, conflict, retry, and failure reason. For bootstrap, the exact SQL-selected set and delivered set are the same security -boundary. The backend always records its policy/profile/source revisions, query -digest, file/byte counts, and completion. A customer policy may additionally +boundary. The backend always records its policy/profile revisions, stable +scopes, attempt-bound source generation vector, query digest, file/byte counts, +and completion. A customer policy may additionally persist exact stable IDs in partitioned `session_deliveries`; default v1 avoids an O(entry count) audit-row write on every launch. A client- or trusted-host materialization receipt is useful telemetry but is not proof of what an @@ -2151,7 +2278,8 @@ be explicit and benchmarked; it must not weaken the ordinary-file default. For a ready source, startup is a streaming pipeline rather than a build job: ```text -session authorization + freshness/revision resolution + query startup +session authorization + export-attempt revalidation/current-head resolution ++ exact metadata query startup + max( selected decoded bytes / database-to-exporter throughput, selected decoded bytes / server tar-and-Zstd throughput, @@ -2175,7 +2303,7 @@ corpus containing many small files. Initial targets, validated per cell/platform rather than treated as vendor promises: -- ready-session authorization, freshness check, revision selection, and query +- ready-session authorization, attempt freshness check, generation selection, and query planning: P95 below 500 ms in-region; - first decoded tar byte: P95 below one second for a warm ready revision, excluding orchestrator scheduling; @@ -2252,14 +2380,14 @@ extraction leaves the prior tree untouched. | ----------------------- | ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ACL/filter query | Large joins, generic plans, or JSON evaluation delay first byte. | Precompute indexed `access_set_id`/subject facts; materialize common filter columns; use a custom fixed query shape; inspect plans on representative cardinalities. | | Metadata/body coupling | Rejected rows still fetch large text/TOAST values. | Keep narrow projection-version rows separate from immutable bodies and join bodies only after exact selection. | -| PostgreSQL storage | Cold/random body reads cannot feed the link. | Prune identical tenant/source hash partitions, batch/order by root/export/storage IDs, use LZ4 TOAST, measure `EXPLAIN (ANALYZE, BUFFERS)`, and add a read replica first. | +| PostgreSQL storage | Cold/random body reads cannot feed the link. | Prune identical tenant/source hash partitions, batch/order by scope/canonical path and immutable storage IDs, use LZ4 TOAST, measure `EXPLAIN (ANALYZE, BUFFERS)`, and add a read replica first. | | Long export transaction | Vacuum pressure or connection starvation. | Bounded session/result limits, statement/idle timeouts, dedicated export pool, cancellation on disconnect, concurrency quotas, and monitoring. | | Exporter network/CPU | TLS/tar/Zstd framing or internal hop limits throughput. | Colocate database/exporter, stream `COPY`/rows without buffering, use Zstd level 1 with bounded buffers, scale stateless exporter processes, and retain `identity` fallback. | | Tiny client | Zstd decode, extraction, or hashing saturates one vCPU. | Concurrent single-frame decode and staged extraction with bounded ring buffers, no metadata import, CPU qualification on the smallest sandbox, and `identity` fallback. | | Many small files | Inode and directory creation dominate bytes. | Report entry count, stage locally, create bounded parallel directories/files only if measured safe, and consider an image fast path later. | | Large attachments | Database or gateway spends resources on opaque binaries. | Store individual immutable attachments in object storage, enforce size/scanning limits, and stream only selected references with backpressure. | | Concurrent sessions | Exports starve ingestion, search, or mutations. | Separate connection pools/worker quotas, per-tenant/cell admission control, read replica routing, and ClickHouse only after the measured trigger. | -| Replica lag | Read replica serves older projection/policy or lacks the new session row. | Bind revision/policy/session state to commit LSNs and route only after replay covers all of them; wait or use primary, never silently downgrade. | +| Replica lag | Read replica serves older projection/policy or lacks the new session/attempt row. | Bind generation/policy/session/attempt state to commit LSNs and route only after replay covers all of them; wait or use primary, never silently downgrade. | | Source lag | Fast database export serves an old provider view. | Freshness gate, pending-event awareness, profile-priority sync, bounded wait/fail behavior, and provider preconditions for every write. | | Repeat-session work | Similar jobs repeatedly read/transmit the same bytes. | Measure first; later cache an exact immutable export under its full policy/profile/source/filter key with TTL and revocation checks. | @@ -2282,7 +2410,9 @@ POST /v1/workspace-profiles/{id}/plan POST /v1/sessions # consumes one-time bootstrap token GET /v1/sessions/{id} -GET /v1/sessions/{id}/export # exact authorized Zstd(tar) stream +POST /v1/sessions/{id}/export-attempts # revalidate, select heads, seal offer +GET /v1/sessions/{id}/export-attempts/{attempt_id}/export + # exact authorized Zstd(tar) stream POST /v1/sessions/{id}/cancel POST /v1/changesets # submits immutable changeset @@ -2305,10 +2435,16 @@ sandbox v1 endpoint. Human-review and advisory-claim endpoints are later extensions to the changeset/session state machines, not v1 placeholders. Session creation validates identity, current policy, profile/runtime filters, -source coverage/freshness, and quotas, then returns accepted revisions and -estimates. The export endpoint accepts no new selectors: it rechecks current -policy/freshness and runs the fixed session query under RLS. A sandbox receives -no database, object-store, KMS, or provider credentials. +stable source scopes, and declared limits, then returns readiness/freshness and +estimates without preparing an archive. Export-attempt creation accepts no new +selectors: it rechecks current authority/freshness/exact quotas under RLS, +selects the current complete source-generation vector, and seals the exact +offer. Its bounded opaque idempotency key is distinct from the server-issued +attempt ID. Reusing the key after an ambiguous POST returns the same attempt, +vector, and offer; a new key creates a new attempt that may select newer heads. +The stream names the server-issued attempt. The current and +previous protocol remain supported during rollout. A sandbox receives no +database, object-store, KMS, or provider credentials. ## Mapping To The Current Codebase @@ -2347,7 +2483,7 @@ one another; both compose lower shared crates. | `locality-connector` | Define host-neutral provider ports and transfer values. It may depend on `locality-core`, but not local storage, daemon, UI, backend, or cloud infrastructure. | | First-party connector crates | Own API clients/DTOs, OAuth metadata, provider retry classification, checkpoint/change observation, native fetch, render/parse/projection, attachment capture, ACL observations, source-specific validation, concurrency checks, apply, and read-back. The same crates run locally or in backend workers. | | `locality-engine` (new) | Own only the shared application workflows described below. It depends on `locality-core` and connector ports, but not SQLite, PostgreSQL, Tauri, filesystem watchers, cloud SDKs, or concrete HTTP transports. | -| `locality-protocol` (new) | Own versioned session, export-metadata, freshness/revision, and changeset wire envelopes. Reuse or wrap domain values; do not create a second canonical identity, operation, or policy model. | +| `locality-protocol` (new) | Own versioned stable source-scope, session, export-attempt/generation-vector, exact-offer/metadata, freshness, and changeset wire envelopes. Reuse or wrap domain values; do not create a second canonical identity, operation, or policy model. | | `locality-store` SQLite | Remain the durable local-host store. Add the small writable-session repository and replica/profile/changeset component versions; do not import read-only entries or make PostgreSQL implement every local repository trait. | | `localityd` | Remain the shared local-host runtime for desktop, headless `loc`, and sandbox use. In direct mode it schedules connectors; in backend mode it bootstraps replicas and submits changesets. In both modes it owns watchers, atomic-write handling, local working-copy safety, status/diff acceleration, local journals, and platform projections. Sandbox v1 has no live-delta loop. | | `loc-cli` | Remain the common command/reporting library and `loc` binary. Commands call local-host/application boundaries and do not duplicate workflows for desktop, direct, or backend mode. The desktop UI may invoke the same library where appropriate. | @@ -2552,7 +2688,7 @@ paths. workspace pinned to an exact public revision. - Define the compact PostgreSQL physical schema with tenant/resource/version/ projection/access-set/checkpoint/revision/idempotency boundaries, narrow - selection rows, separate body rows, source/root/export/storage locality keys, + selection rows, separate body rows, source/scope/path/storage locality keys, the four baseline index shapes, and identical tenant/source hash partitioning for only the three serving tables. Compare unpartitioned and small partition-count fixtures and record the initial cell default before production @@ -2581,7 +2717,9 @@ streams without O(read-only entry count) SQLite writes. ECS execution roles and from the schema-owner migrator. - Implement tenants, real non-owner RLS, principals/groups, broad pilot DataGrants, WorkspaceProfiles, access sets/subjects, content/projection - versions, ready revisions, sessions, PostgreSQL jobs, audit, and retention. + versions, stable source scopes/memberships, ready generations, + scope-authorized sessions, attempt-bound generation vectors/offers, + PostgreSQL jobs, audit, and retention. Create the three serving tables with the selected fixed hash partitions, baseline indexes, LZ4 body compression where available, and table-specific fillfactor defaults; leave all other tables unpartitioned. @@ -2591,8 +2729,9 @@ streams without O(read-only entry count) SQLite writes. - Implement resumable profile-prioritized backfill, scheduled sync/repair, immutable version creation, complete ready-revision publication, and the freshness wait/fail gate. Batch initial projection/content inserts by - source/root/export order and analyze large completed backfills. -- Implement the exact parameterized ACL/profile/runtime selection query, + source/scope/canonical-path order and analyze large completed backfills. +- Implement the exact parameterized current-head scope/ACL/profile/runtime + selection query, overlapping-scope deduplication without action composition, negotiated Zstd-or-identity streaming tar endpoint, bounded attachment resolution, concurrent bounded decoder/safe staged extractor, and atomic tree publication. @@ -2603,8 +2742,10 @@ streams without O(read-only entry count) SQLite writes. limits. - Exchange a one-time orchestrator bootstrap token for a short-lived capability and ship token-only `loc sandbox init` with a plain-file cloud mount. Profile, - actor, workload, roots, actions, and runtime narrowing are sealed before token - issuance and cannot be supplied again by the sandbox command. + actor, workload, stable scopes, actions, limits, and runtime narrowing are + sealed before token issuance and cannot be supplied again by the sandbox + command. Export-attempt preflight selects current complete heads and seals the + exact offer; it is not performed during bootstrap exchange. - Enable RDS automated backups/PITR and S3 versioning/lifecycle, declare RPO/RTO, and complete an isolated restore drill before storing production customer data. Recovery verifies database state, attachments, secret references, @@ -2635,7 +2776,7 @@ cell and schema. - Qualify cold/warm 1/10/50-session export throughput, database/exporter resource isolation, TOAST/body behavior, cancellation, quotas, and primary routing. Add a PostgreSQL read replica only if these measurements require it; gate routing - by the selected revision/policy and session commit LSNs. + by the selected generations/policy and session/attempt commit LSNs. Exit: several customer tenants mount at least two structurally different sources with independent watermarks. Remote edits converge even without webhook hints; @@ -2712,6 +2853,13 @@ verification, and dedicated-cell operational runbooks are complete. - Old session/replica/export/profile/component versions open or migrate. - Newer required versions fail with `NeedsUpdate` and preserve pending work. - Backend serves current and previous negotiated export readers during rollout. +- Existing v1 projection-pinned sessions always use their exact historical query + path and offer semantics; they are never reinterpreted as scope-authorized + current-head sessions. New issuance switches to v2 only after the v2 client, + protocol, repository, and endpoint are deployed together. +- Retain legacy authority tables/readers for at least the maximum v1 session and + ambiguous-recovery TTL. Emit versioned use metrics, require zero live use, and + remove them only in a later forward cleanup migration with rollback evidence. - Direct-to-backend cutover preserves dirty/conflicted files, journals, virtual creates/moves/deletes, writable baselines, and apply effects. - The private backend builds against its exact public Git revision/Cargo lock; @@ -2778,27 +2926,42 @@ verification, and dedicated-cell operational runbooks are complete. connector cooldown, auth failure, or replica backlog produces bounded wait and explicit bootstrapping/stale/unavailable outcomes—not a silent older export. - Updates immediately before, during, and after an export prove one repeatable- - read response contains one consistent accepted view. + read metadata selection contains one consistent accepted generation vector; + a stale/expired publisher cannot replace a newer head. ### PostgreSQL Export And Client - Exact query golden fixtures across broad grants, per-resource access sets, multiple groups/acting principals, roots, path/time/kind filters, exclusions, multiple sources, moves, tombstones, and profile revisions. +- Scope fixtures cover parent/child and many-to-many overlap, configured scope + ordinal, one output/body fetch per projection, and proof that partial actions + from different scopes are never combined. - Schema/plan fixtures prove every serving-table access carries tenant/source, prunes the expected hash partitions, uses current-head versus historical revision paths correctly, and joins bodies by `content_storage_id` only after authorization/filter selection. -- Ordered-ingest/export fixtures prove related source/root rows receive monotonic - export/storage IDs and the tar's entry order does not change logical paths or - canonical bytes. LZ4-storage and default-storage fixtures return identical - plaintext. +- Ordered-ingest/export fixtures prove the single reserved control member is + first and counted, directory headers then use parent-first `(depth, + logical_path)` order, and file members use + `(winning_scope_ordinal, parent_path, logical_path, projection_id)` under + bytewise `C` collation. The order does not change logical paths or canonical + bytes. LZ4-storage and default-storage fixtures return identical plaintext. - Query-plan tests prove authorization precedes limit/ranking/body fetch and rejected entries do not disclose path, title, size, count, timing-dependent top-K membership, or plaintext. - Tampered runtime values, unsupported operators/fields, SQL metacharacters, excessive limits, a session/profile mismatch, and direct export reuse after revocation are rejected. +- Four authorized roots create four capability/session scope rows and zero + per-projection authority rows. A new descendant appears in a new attempt + without session reissuance. Retry after process restart remains on generation + A after B publishes, including old memberships/actions/paths/S3 versions; + active-attempt retention prevents premature garbage collection. +- Credential-ceiling, provider ACL/access-set, scope, grant, group, or principal + revocation; stale heads; and exact entry/byte/attachment quota excess fail + both new attempts and old-attempt retries before any body reference or S3 + object is opened. - Real non-owner RLS tests cover cross-tenant IDs/content/search/session queries, owner/`BYPASSRLS` regressions, transaction-pool context leakage, and repository predicates. @@ -2807,6 +2970,10 @@ verification, and dedicated-cell operational runbooks are complete. collision, Unicode normalization, reserved metadata path, entry/byte/disk limits, disconnect cancellation, bounded pipeline backpressure, and atomic rollback. +- Attempt-state tests cover successful selected-equals-delivered completion, + disconnect, cancellation, exact retry, concurrent replay, and database + commit-response ambiguity. Every non-success terminal state records exact + selected/delivered counts without exposing a partial tree. - Reserved writable metadata creates SQLite rows/baselines only for exact write-enabled entries; read-only entries create no shadow/three-tree/entity state and cannot become changeset operations. @@ -2823,7 +2990,7 @@ verification, and dedicated-cell operational runbooks are complete. Run 1M-entry/read-replica/cache/ClickHouse qualifications only when those scales/features are targeted. - When a read replica is enabled, exports wait/use primary until replay LSN - covers selected revision/policy and session-creation commits; they never + covers selected generation/policy and session/attempt commits; they never silently serve older state or miss the session row. ### Policy And Security @@ -2973,9 +3140,11 @@ queries, channel names, source content, or runtime filter values. - **Persist every delivered entry by default:** exact inventory rows can be policy-enabled for governance, but default counts/digests avoid an O(entry count) write on every read-heavy launch. -- **Stream live deltas into every ephemeral sandbox:** an immutable start view - plus provider preconditions gives v1 correctness without rewriting files under - an agent. Live revisions remain a desktop/long-session extension. +- **Stream live deltas into every ephemeral sandbox:** an immutable tree from one + export attempt plus provider preconditions gives v1 correctness without + rewriting files under an agent. A refresh uses a new current-head attempt and + atomic tree replacement; in-place live revisions remain a desktop/long-session + extension. ## Decisions To Validate, Not Reopen By Default @@ -3017,9 +3186,10 @@ architecture: behavior; and - which sandbox products expose a safe post-cache, pre-agent injection hook. -The architectural baseline is central continuous ingestion; complete ready -revisions with explicit freshness; one managed PostgreSQL correctness and v1 -read-serving plane; exact server-side ACL/profile filtering; a transient +The architectural baseline is central continuous ingestion; atomically published +complete ready generations with explicit freshness; stable-scope session +authority plus attempt-bound current generation vectors; one managed PostgreSQL +correctness and v1 read-serving plane; exact server-side ACL/profile filtering; a transient Zstd-compressed streaming tar with `identity` fallback rather than stored packs; object storage only for large attachments/explicit exports; managed database snapshots/PITR; ordinary local files; writable-only three-tree state; From 80dba5e4b170cc208915d9096af4b3e93d9490cd Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 15:07:12 -0700 Subject: [PATCH 02/14] Specify one-pass terminal export receipts --- docs/cloud-sandbox-data-plane.md | 39 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/docs/cloud-sandbox-data-plane.md b/docs/cloud-sandbox-data-plane.md index ea3cd6e1..e86828ce 100644 --- a/docs/cloud-sandbox-data-plane.md +++ b/docs/cloud-sandbox-data-plane.md @@ -1308,43 +1308,42 @@ Content-Encoding: zstd `identity` remains a required fallback and benchmark control. In either mode: - the tar is generated on demand and is never stored as a canonical artifact; -- the sealed v2 offer distinguishes one control member, file count, directory - count, total archive entry count, selected content bytes, and a canonical - inventory digest so - parent directory headers cannot create an ambiguous count. It does not +- the sealed v2 offer distinguishes file count, directory count, one terminal + control member, total archive entry count, selected content bytes, and a + canonical inventory digest so parent directory headers cannot create an + ambiguous count. It does not pre-render tar/Zstd merely to predict encoded length or stream digest; the - completion receipt records actual decoded/encoded bytes and stream digest - after the one streaming pass; -- the reserved writable-metadata control member is archive record class 0 and - is always first, even when its writable-entry list is empty. It participates - in total archive-entry count and the canonical inventory digest; -- directory headers are record class 1 and follow in + completion receipt records actual delivered file/content counts and a + domain-separated delivered-body digest after the one streaming pass; +- directory headers are record class 0 and follow in `(depth, logical_path COLLATE "C")` order, so every unique parent precedes descendants; -- each selected projection is record class 2 and becomes one ordinary tar member with its logical +- each selected projection is record class 1 and becomes one ordinary tar member with its logical path, size, and non-executable mode. File members follow `(winning_scope_ordinal, parent_path COLLATE "C", logical_path COLLATE "C", projection_id)` order; -- a reserved first member contains versioned session metadata only for writable - paths: projection/resource identity, delivered content hash, provider - precondition, effective actions, and baseline requirements. The client consumes - this member into its small SQLite store and does not expose it in the mount; +- v1 retains its reserved first writable-metadata member unchanged. V2 instead + requires exactly one final record-class-2 `.loc/session.json` member combining + that writable metadata with the completion receipt. It is bounded, canonical + JSON, regular mode `0444`, consumed into the small SQLite store, never exposed + in the mount, and has no emitted `.loc/` directory header; - read-only entries create no local metadata or shadow rows; - selected large attachment references are fetched by the exporter and appended to the same stream under bounded size/scanning policy; and - the client validates and extracts into a staging directory with path, link, device, case, Unicode, entry-count, byte, and disk limits, then atomically publishes the tree only after the HTTP response, Zstd frame, tar stream, and - file writes terminate cleanly and its recomputed canonical inventory digest - equals the sealed offer. + file writes terminate cleanly, the final receipt is present once with no + following member, and its recomputed canonical inventory/body digests equal + the sealed offer/receipt. The v2 canonical inventory digest is SHA-256 over a domain-separated, length-framed binary preimage. It begins with `locality.export.inventory.v2\0`; every scalar is an unsigned 64-bit big-endian length followed by its UTF-8 or raw bytes, and every list begins with an unsigned -64-bit big-endian count. Records follow the archive's class/order above. The -control record includes its path plus exact body SHA-256 and length; a directory -includes its normalized `LogicalPath`; a file includes global winning scope +64-bit big-endian count. Only directory and file records participate, in their +archive order; the final self-describing receipt is deliberately excluded to +avoid a circular digest. A directory includes its normalized `LogicalPath`; a file includes global winning scope ordinal, source connection ID, projection ID, normalized logical path, file kind, sorted effective-action labels, content SHA-256, and byte length. Public exact goldens bind the preimage and digest so the backend and client cannot disagree From fac9f6a402000502e0bc482f4317d29ed696bb6b Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 15:17:51 -0700 Subject: [PATCH 03/14] feat(protocol): add scope-authorized export v2 contracts --- Cargo.lock | 1 + crates/locality-core/src/portable.rs | 62 +- crates/locality-protocol/Cargo.toml | 1 + .../fixtures/canonical-export-inventory.json | 4 + .../fixtures/canonical-export-records.json | 56 + .../fixtures/export-attempt-request.json | 19 + .../fixtures/export-completion-receipt.json | 28 + .../scope-authorized-session-query.json | 54 + .../fixtures/sealed-export-offer.json | 35 + crates/locality-protocol/src/lib.rs | 1079 ++++++++++++++++- crates/locality-protocol/tests/golden.rs | 358 +++++- 11 files changed, 1669 insertions(+), 28 deletions(-) create mode 100644 crates/locality-protocol/fixtures/canonical-export-inventory.json create mode 100644 crates/locality-protocol/fixtures/canonical-export-records.json create mode 100644 crates/locality-protocol/fixtures/export-attempt-request.json create mode 100644 crates/locality-protocol/fixtures/export-completion-receipt.json create mode 100644 crates/locality-protocol/fixtures/scope-authorized-session-query.json create mode 100644 crates/locality-protocol/fixtures/sealed-export-offer.json diff --git a/Cargo.lock b/Cargo.lock index c0f31cd4..25fa3486 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2298,6 +2298,7 @@ dependencies = [ "locality-core", "serde", "serde_json", + "sha2", ] [[package]] diff --git a/crates/locality-core/src/portable.rs b/crates/locality-core/src/portable.rs index 35210ee3..12610636 100644 --- a/crates/locality-core/src/portable.rs +++ b/crates/locality-core/src/portable.rs @@ -46,6 +46,64 @@ opaque_id!(ReplicaRevisionId); opaque_id!(ChangesetId); opaque_id!(AccessSetId); +macro_rules! validated_opaque_id { + ($name:ident) => { + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] + #[serde(transparent)] + pub struct $name(String); + + impl $name { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if value.is_empty() { + return Err(PortableIdError::Empty); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(serde::de::Error::custom) + } + } + + impl Display for $name { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.0) + } + } + }; +} + +/// A validated opaque portable identifier cannot be the empty string. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PortableIdError { + Empty, +} + +impl Display for PortableIdError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Empty => formatter.write_str("portable identifier is empty"), + } + } +} + +impl std::error::Error for PortableIdError {} + +validated_opaque_id!(SourceScopeId); +validated_opaque_id!(ExportAttemptId); +validated_opaque_id!(SourceGenerationId); + /// The only reserved export member in the portable path namespace. /// /// The host consumes this member as writable-session metadata and never @@ -53,7 +111,9 @@ opaque_id!(AccessSetId); /// connector-managed artifacts such as media. pub const RESERVED_EXPORT_METADATA_PATH: &str = ".loc/session.json"; -#[derive(Clone, Debug, PartialEq, Eq)] +/// A validated portable path. Its ordering is canonical bytewise UTF-8 order, +/// matching PostgreSQL `COLLATE "C"` for the same encoded values. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct LogicalPath(String); impl LogicalPath { diff --git a/crates/locality-protocol/Cargo.toml b/crates/locality-protocol/Cargo.toml index 143840ce..02ea8bef 100644 --- a/crates/locality-protocol/Cargo.toml +++ b/crates/locality-protocol/Cargo.toml @@ -13,6 +13,7 @@ path = "src/lib.rs" [dependencies] locality-core.workspace = true serde = { version = "1.0", features = ["derive"] } +sha2 = "0.10" [dev-dependencies] serde_json = "1.0" diff --git a/crates/locality-protocol/fixtures/canonical-export-inventory.json b/crates/locality-protocol/fixtures/canonical-export-inventory.json new file mode 100644 index 00000000..a186f04e --- /dev/null +++ b/crates/locality-protocol/fixtures/canonical-export-inventory.json @@ -0,0 +1,4 @@ +{ + "preimage_hex": "6c6f63616c6974792e6578706f72742e696e76656e746f72792e763200000000000000000400000000000000096469726563746f7279000000000000000850726f6a6563747300000000000000096469726563746f7279000000000000001050726f6a656374732f526f61646d6170000000000000000466696c6500000000000000080000000000000000000000000000000d736f757263652d6e6f74696f6e000000000000001270726f6a656374696f6e2d726f61646d6170000000000000001850726f6a656374732f526f61646d61702f706167652e6d6400000000000000086d61726b646f776e0000000000000002000000000000000472656164000000000000000675706461746500000000000000477368613235363a353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535350000000000000008000000000000000a000000000000000466696c6500000000000000080000000000000001000000000000000d736f757263652d6e6f74696f6e000000000000001170726f6a656374696f6e2d726561646d650000000000000009524541444d452e6d6400000000000000086d61726b646f776e000000000000000100000000000000047265616400000000000000477368613235363a3636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363600000000000000080000000000000007", + "sha256": "sha256:025cdbae136931542f7fa881da423e8e1f29a6132cf26ae5f4eea53c53a8ef51" +} diff --git a/crates/locality-protocol/fixtures/canonical-export-records.json b/crates/locality-protocol/fixtures/canonical-export-records.json new file mode 100644 index 00000000..9822b86a --- /dev/null +++ b/crates/locality-protocol/fixtures/canonical-export-records.json @@ -0,0 +1,56 @@ +[ + { + "record_class": "directory", + "order_key": { + "depth": 1, + "logical_path": "Projects" + } + }, + { + "record_class": "directory", + "order_key": { + "depth": 2, + "logical_path": "Projects/Roadmap" + } + }, + { + "record_class": "file", + "order_key": { + "winning_scope_ordinal": 0, + "parent_path": "Projects/Roadmap", + "logical_path": "Projects/Roadmap/page.md", + "projection_id": "projection-roadmap" + }, + "source_connection_id": "source-notion", + "file_kind": "markdown", + "effective_actions": [ + "read", + "update" + ], + "content_sha256": "sha256:5555555555555555555555555555555555555555555555555555555555555555", + "byte_length": 10 + }, + { + "record_class": "file", + "order_key": { + "winning_scope_ordinal": 1, + "parent_path": null, + "logical_path": "README.md", + "projection_id": "projection-readme" + }, + "source_connection_id": "source-notion", + "file_kind": "markdown", + "effective_actions": [ + "read" + ], + "content_sha256": "sha256:6666666666666666666666666666666666666666666666666666666666666666", + "byte_length": 7 + }, + { + "record_class": "control", + "order_key": { + "ordinal": 0 + }, + "member_path": ".loc/session.json" + } +] diff --git a/crates/locality-protocol/fixtures/export-attempt-request.json b/crates/locality-protocol/fixtures/export-attempt-request.json new file mode 100644 index 00000000..923b9baf --- /dev/null +++ b/crates/locality-protocol/fixtures/export-attempt-request.json @@ -0,0 +1,19 @@ +{ + "versions": { + "session": 2, + "replica": 2, + "export_metadata": 2, + "writable_session_store": 1, + "canonical": 1, + "path": 1, + "changeset": 1 + }, + "opaque_session_capability": "opaque-session-capability", + "idempotency_key": "mount-attempt-2026-07-23T19:00:00Z", + "content_encoding": "zstd", + "limits": { + "max_files": 10000, + "max_directories": 10000, + "max_content_bytes": 104857600 + } +} diff --git a/crates/locality-protocol/fixtures/export-completion-receipt.json b/crates/locality-protocol/fixtures/export-completion-receipt.json new file mode 100644 index 00000000..8e49b4d4 --- /dev/null +++ b/crates/locality-protocol/fixtures/export-completion-receipt.json @@ -0,0 +1,28 @@ +{ + "versions": { + "session": 2, + "replica": 2, + "export_metadata": 2, + "writable_session_store": 1, + "canonical": 1, + "path": 1, + "changeset": 1 + }, + "session_id": "session-scope-7", + "export_attempt_id": "export-attempt-9", + "source_generations": [ + { + "ordinal": 0, + "source_connection_id": "source-notion", + "source_generation_id": "generation-notion-109" + } + ], + "inventory_sha256": "sha256:025cdbae136931542f7fa881da423e8e1f29a6132cf26ae5f4eea53c53a8ef51", + "delivered_control_entry_count": 1, + "delivered_file_count": 2, + "delivered_directory_count": 2, + "delivered_archive_entry_count": 5, + "delivered_content_bytes": 17, + "delivered_body_sha256": "sha256:d3dc0915a66e57487d402df0642649990fc61e6128ca8a275dea1e7da93e1381", + "completed_at": "2026-07-23T19:00:04Z" +} diff --git a/crates/locality-protocol/fixtures/scope-authorized-session-query.json b/crates/locality-protocol/fixtures/scope-authorized-session-query.json new file mode 100644 index 00000000..a1cf1948 --- /dev/null +++ b/crates/locality-protocol/fixtures/scope-authorized-session-query.json @@ -0,0 +1,54 @@ +{ + "versions": { + "session": 2, + "replica": 2, + "export_metadata": 2, + "writable_session_store": 1, + "canonical": 1, + "path": 1, + "changeset": 1 + }, + "tenant_id": "tenant-acme", + "session_id": "session-scope-7", + "acting_principal_id": "principal-agent", + "workload_id": "workload-sandbox", + "authorization_revision": 43, + "policy_revision": 18, + "profile_revision": 10, + "authorized_scopes": [ + { + "ordinal": 0, + "source_scope_id": "scope-product", + "source_connection_id": "source-notion", + "selector": { + "provider": "notion", + "selector_version": 1, + "scope_kind": "page", + "provider_scope_id": "e07dd2a2531444aba2f452010314fb87" + }, + "effective_actions": [ + "read", + "download_attachment" + ], + "validated_filter_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111" + }, + { + "ordinal": 1, + "source_scope_id": "scope-go-to-market", + "source_connection_id": "source-notion", + "selector": { + "provider": "notion", + "selector_version": 1, + "scope_kind": "page", + "provider_scope_id": "f525c3a7db684465a9e1d1082bea6f81" + }, + "effective_actions": [ + "read" + ], + "validated_filter_digest": null + } + ], + "max_files": 10000, + "max_directories": 10000, + "max_bytes": 104857600 +} diff --git a/crates/locality-protocol/fixtures/sealed-export-offer.json b/crates/locality-protocol/fixtures/sealed-export-offer.json new file mode 100644 index 00000000..0100fa18 --- /dev/null +++ b/crates/locality-protocol/fixtures/sealed-export-offer.json @@ -0,0 +1,35 @@ +{ + "versions": { + "session": 2, + "replica": 2, + "export_metadata": 2, + "writable_session_store": 1, + "canonical": 1, + "path": 1, + "changeset": 1 + }, + "session_id": "session-scope-7", + "export_attempt_id": "export-attempt-9", + "source_generations": [ + { + "ordinal": 0, + "source_connection_id": "source-notion", + "source_generation_id": "generation-notion-109" + } + ], + "media_type": "application/x-tar", + "content_encoding": "zstd", + "limits": { + "max_files": 10000, + "max_directories": 10000, + "max_content_bytes": 104857600 + }, + "control_entry_count": 1, + "file_count": 2, + "directory_count": 2, + "archive_entry_count": 5, + "selected_content_bytes": 17, + "inventory_sha256": "sha256:025cdbae136931542f7fa881da423e8e1f29a6132cf26ae5f4eea53c53a8ef51", + "sealed_at": "2026-07-23T19:00:01Z", + "expires_at": "2026-07-23T19:10:01Z" +} diff --git a/crates/locality-protocol/src/lib.rs b/crates/locality-protocol/src/lib.rs index 6e080f4f..2b0e1df0 100644 --- a/crates/locality-protocol/src/lib.rs +++ b/crates/locality-protocol/src/lib.rs @@ -10,12 +10,14 @@ use std::fmt::{Debug, Display, Formatter}; use locality_core::journal::PushOperationId; use locality_core::model::RemoteId; use locality_core::portable::{ - AccessSetId, ChangesetId, ContentVersionId, LogicalPath, PrincipalId, ProjectionEntry, - ProjectionFileKind, ProjectionId, ProjectionVersionId, ReplicaRevisionId, SessionId, - SourceAction, SourceConnectionId, SourceOperationPlan, SourceVersionId, TenantId, + AccessSetId, ChangesetId, ContentVersionId, ExportAttemptId, LogicalPath, PrincipalId, + ProjectionEntry, ProjectionFileKind, ProjectionId, ProjectionVersionId, ReplicaRevisionId, + SessionId, SourceAction, SourceConnectionId, SourceGenerationId, SourceOperationPlan, + SourceScopeId, SourceVersionId, TenantId, }; use locality_core::readable_diff::ReadableDiffOutput; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; pub use locality_core::portable::RESERVED_EXPORT_METADATA_PATH; @@ -29,6 +31,20 @@ pub const COMPONENT_VERSIONS: ComponentVersions = ComponentVersions { changeset: 1, }; +/// Latest component versions this crate can decode and validate. +/// +/// [`COMPONENT_VERSIONS`] remains the byte-compatible v1 emission default. +/// Callers opt into the scope-authorized export contracts explicitly. +pub const LATEST_COMPONENT_VERSIONS: ComponentVersions = ComponentVersions { + session: 2, + replica: 2, + export_metadata: 2, + ..COMPONENT_VERSIONS +}; + +/// Component versions required by scope-authorized current-head exports. +pub const SCOPE_AUTHORIZED_COMPONENT_VERSIONS: ComponentVersions = LATEST_COMPONENT_VERSIONS; + pub const MINIMUM_COMPONENT_VERSIONS: ComponentVersions = COMPONENT_VERSIONS; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -49,43 +65,43 @@ impl ComponentVersions { ProtocolComponent::Session, self.session, MINIMUM_COMPONENT_VERSIONS.session, - COMPONENT_VERSIONS.session, + LATEST_COMPONENT_VERSIONS.session, ), ( ProtocolComponent::Replica, self.replica, MINIMUM_COMPONENT_VERSIONS.replica, - COMPONENT_VERSIONS.replica, + LATEST_COMPONENT_VERSIONS.replica, ), ( ProtocolComponent::ExportMetadata, self.export_metadata, MINIMUM_COMPONENT_VERSIONS.export_metadata, - COMPONENT_VERSIONS.export_metadata, + LATEST_COMPONENT_VERSIONS.export_metadata, ), ( ProtocolComponent::WritableSessionStore, self.writable_session_store, MINIMUM_COMPONENT_VERSIONS.writable_session_store, - COMPONENT_VERSIONS.writable_session_store, + LATEST_COMPONENT_VERSIONS.writable_session_store, ), ( ProtocolComponent::Canonical, self.canonical, MINIMUM_COMPONENT_VERSIONS.canonical, - COMPONENT_VERSIONS.canonical, + LATEST_COMPONENT_VERSIONS.canonical, ), ( ProtocolComponent::Path, self.path, MINIMUM_COMPONENT_VERSIONS.path, - COMPONENT_VERSIONS.path, + LATEST_COMPONENT_VERSIONS.path, ), ( ProtocolComponent::Changeset, self.changeset, MINIMUM_COMPONENT_VERSIONS.changeset, - COMPONENT_VERSIONS.changeset, + LATEST_COMPONENT_VERSIONS.changeset, ), ] { if required > supported { @@ -246,6 +262,273 @@ pub struct SessionReplicaRevision { pub replica_revision_id: ReplicaRevisionId, } +/// The provider-specific meaning of one stable source scope. +/// +/// Each variant uses stable provider IDs. Titles, projected paths, and search +/// text are deliberately absent because they cannot establish membership. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "provider", rename_all = "snake_case")] +pub enum ProviderSourceScopeSelector { + Notion { + selector_version: u16, + scope_kind: NotionScopeKind, + provider_scope_id: String, + }, + Slack { + selector_version: u16, + conversation_id: String, + }, + Granola { + selector_version: u16, + scope_kind: GranolaScopeKind, + provider_scope_id: String, + }, + Gmail { + selector_version: u16, + mailbox_id: String, + scope_kind: GmailScopeKind, + provider_scope_id: String, + }, + GoogleDrive { + selector_version: u16, + scope_kind: GoogleDriveScopeKind, + provider_scope_id: String, + }, + Github { + selector_version: u16, + scope_kind: GithubScopeKind, + provider_scope_id: String, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NotionScopeKind { + Page, + Database, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GranolaScopeKind { + Collection, + Team, + Folder, + Meeting, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GmailScopeKind { + Label, + Thread, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GoogleDriveScopeKind { + SharedDrive, + Folder, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GithubScopeKind { + Organization, + Repository, +} + +impl ProviderSourceScopeSelector { + pub fn validate(&self) -> Result<(), ScopeContractError> { + let selector_version = match self { + Self::Notion { + selector_version, .. + } + | Self::Slack { + selector_version, .. + } + | Self::Granola { + selector_version, .. + } + | Self::Gmail { + selector_version, .. + } + | Self::GoogleDrive { + selector_version, .. + } + | Self::Github { + selector_version, .. + } => *selector_version, + }; + if selector_version != 1 { + return Err(ScopeContractError::UnsupportedSelectorVersion { + version: selector_version, + }); + } + + match self { + Self::Notion { + provider_scope_id, .. + } + | Self::Granola { + provider_scope_id, .. + } + | Self::GoogleDrive { + provider_scope_id, .. + } + | Self::Github { + provider_scope_id, .. + } => validate_nonempty("provider_scope_id", provider_scope_id)?, + Self::Slack { + conversation_id, .. + } => validate_nonempty("conversation_id", conversation_id)?, + Self::Gmail { + mailbox_id, + provider_scope_id, + .. + } => { + validate_nonempty("mailbox_id", mailbox_id)?; + validate_nonempty("provider_scope_id", provider_scope_id)?; + } + } + Ok(()) + } +} + +/// One source scope after provider, tenant, principal, workload, and profile +/// authority have been intersected. `ordinal` is the profile-configured order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuthorizedSourceScope { + pub ordinal: u32, + pub source_scope_id: SourceScopeId, + pub source_connection_id: SourceConnectionId, + pub selector: ProviderSourceScopeSelector, + pub effective_actions: BTreeSet, + pub validated_filter_digest: Option, +} + +impl AuthorizedSourceScope { + pub fn validate(&self) -> Result<(), ScopeContractError> { + validate_nonempty("source_connection_id", self.source_connection_id.as_str())?; + self.selector.validate()?; + if self.effective_actions.is_empty() { + return Err(ScopeContractError::EmptyCollection("effective_actions")); + } + if let Some(digest) = &self.validated_filter_digest { + validate_sha256("validated_filter_digest", digest)?; + } + Ok(()) + } +} + +/// An authorized current-head query. Unlike [`AuthorizedSessionQuery`], this +/// value names stable source scopes and deliberately does not pin revisions. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ScopeAuthorizedSessionQuery { + pub versions: ComponentVersions, + pub tenant_id: TenantId, + pub session_id: SessionId, + pub acting_principal_id: PrincipalId, + pub workload_id: String, + pub authorization_revision: u64, + pub policy_revision: u64, + pub profile_revision: u64, + pub authorized_scopes: Vec, + pub max_files: u64, + pub max_directories: u64, + pub max_bytes: u64, +} + +impl ScopeAuthorizedSessionQuery { + pub fn validate(&self) -> Result<(), ScopeContractError> { + self.versions.validate_required()?; + if self.versions.session < 2 { + return Err(ScopeContractError::ComponentVersionTooOld { + component: ProtocolComponent::Session, + required: 2, + actual: self.versions.session, + }); + } + if self.versions.replica < 2 { + return Err(ScopeContractError::ComponentVersionTooOld { + component: ProtocolComponent::Replica, + required: 2, + actual: self.versions.replica, + }); + } + validate_nonempty("tenant_id", self.tenant_id.as_str())?; + validate_nonempty("session_id", self.session_id.as_str())?; + validate_nonempty("acting_principal_id", self.acting_principal_id.as_str())?; + validate_nonempty("workload_id", &self.workload_id)?; + if self.authorized_scopes.is_empty() { + return Err(ScopeContractError::EmptyCollection("authorized_scopes")); + } + if self.max_files == 0 || self.max_directories == 0 || self.max_bytes == 0 { + return Err(ScopeContractError::InvalidLimit); + } + + let mut scope_ids = BTreeSet::new(); + for (expected_ordinal, scope) in self.authorized_scopes.iter().enumerate() { + scope.validate()?; + if scope.ordinal as usize != expected_ordinal { + return Err(ScopeContractError::NonCanonicalOrdinal { + collection: "authorized_scopes", + expected: expected_ordinal as u32, + actual: scope.ordinal, + }); + } + if !scope_ids.insert(scope.source_scope_id.clone()) { + return Err(ScopeContractError::DuplicateValue("source_scope_id")); + } + } + Ok(()) + } +} + +/// Compatibility decoder for already-issued revision-pinned sessions and new +/// scope-authorized current-head sessions. Variants remain structurally +/// distinct on the wire and are never silently reinterpreted. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum CompatibleAuthorizedSessionQuery { + Scope(ScopeAuthorizedSessionQuery), + Legacy(AuthorizedSessionQuery), +} + +/// One source generation in configured source order for an export attempt. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct OrderedSourceGeneration { + pub ordinal: u32, + pub source_connection_id: SourceConnectionId, + pub source_generation_id: SourceGenerationId, +} + +pub fn validate_source_generations( + source_generations: &[OrderedSourceGeneration], +) -> Result<(), ScopeContractError> { + if source_generations.is_empty() { + return Err(ScopeContractError::EmptyCollection("source_generations")); + } + let mut source_ids = BTreeSet::new(); + for (expected_ordinal, generation) in source_generations.iter().enumerate() { + if generation.ordinal as usize != expected_ordinal { + return Err(ScopeContractError::NonCanonicalOrdinal { + collection: "source_generations", + expected: expected_ordinal as u32, + actual: generation.ordinal, + }); + } + validate_nonempty( + "source_connection_id", + generation.source_connection_id.as_str(), + )?; + if !source_ids.insert(generation.source_connection_id.clone()) { + return Err(ScopeContractError::DuplicateValue("source_connection_id")); + } + } + Ok(()) +} + /// One logical row from the exact authorized query. /// /// The exporter yields these rows in its authorized stream order. Physical @@ -266,6 +549,286 @@ pub struct OrderedExportRow { pub body: Vec, } +/// Canonical key for the single final metadata/receipt control member. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct CanonicalControlOrderKey { + pub ordinal: u8, +} + +/// Canonical first-phase order for a directory tar record. `depth` counts the +/// directory itself, so `Projects` is depth 1 and `Projects/Roadmap` is 2. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct CanonicalDirectoryOrderKey { + pub depth: u32, + pub logical_path: LogicalPath, +} + +/// Canonical second-phase order for a projected file tar record. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct CanonicalFileOrderKey { + pub winning_scope_ordinal: u32, + pub parent_path: Option, + pub logical_path: LogicalPath, + pub projection_id: ProjectionId, +} + +/// The enum discriminant makes directories sort before files and the single +/// reserved control member sort last. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(tag = "record_class", content = "order_key", rename_all = "snake_case")] +pub enum CanonicalExportOrderKey { + Directory(CanonicalDirectoryOrderKey), + File(CanonicalFileOrderKey), + Control(CanonicalControlOrderKey), +} + +/// Metadata-only record used to validate and order an export before body reads. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "record_class", rename_all = "snake_case")] +pub enum CanonicalExportRecord { + Directory { + order_key: CanonicalDirectoryOrderKey, + }, + File { + order_key: CanonicalFileOrderKey, + source_connection_id: SourceConnectionId, + file_kind: ProjectionFileKind, + effective_actions: BTreeSet, + content_sha256: String, + byte_length: u64, + }, + Control { + order_key: CanonicalControlOrderKey, + member_path: String, + }, +} + +impl CanonicalExportRecord { + pub fn order_key(&self) -> CanonicalExportOrderKey { + match self { + Self::Control { order_key, .. } => CanonicalExportOrderKey::Control(order_key.clone()), + Self::Directory { order_key } => CanonicalExportOrderKey::Directory(order_key.clone()), + Self::File { order_key, .. } => CanonicalExportOrderKey::File(order_key.clone()), + } + } + + pub fn validate(&self) -> Result<(), ScopeContractError> { + match self { + Self::Control { + order_key, + member_path, + } => { + if order_key.ordinal != 0 || member_path != RESERVED_EXPORT_METADATA_PATH { + return Err(ScopeContractError::InvalidControlRecord); + } + } + Self::Directory { order_key } => { + if order_key.logical_path.as_str().eq_ignore_ascii_case(".loc") { + return Err(ScopeContractError::InvalidControlDirectory); + } + let actual_depth = path_depth(&order_key.logical_path); + if order_key.depth != actual_depth { + return Err(ScopeContractError::InvalidDirectoryDepth { + expected: actual_depth, + actual: order_key.depth, + }); + } + } + Self::File { + order_key, + source_connection_id, + effective_actions, + content_sha256, + .. + } => { + let expected_parent = logical_parent(&order_key.logical_path)?; + if order_key.parent_path != expected_parent { + return Err(ScopeContractError::InvalidParentPath); + } + validate_nonempty("source_connection_id", source_connection_id.as_str())?; + if effective_actions.is_empty() { + return Err(ScopeContractError::EmptyCollection("effective_actions")); + } + validate_sha256("content_sha256", content_sha256)?; + } + } + Ok(()) + } +} + +pub fn validate_canonical_export_records( + records: &[CanonicalExportRecord], +) -> Result<(), ScopeContractError> { + let mut control_count = 0_u64; + let mut logical_paths = BTreeSet::new(); + let mut directory_paths = BTreeSet::new(); + let mut previous_key = None; + for record in records { + record.validate()?; + if matches!(record, CanonicalExportRecord::Control { .. }) { + control_count += 1; + } + let logical_path = match record { + CanonicalExportRecord::Control { .. } => None, + CanonicalExportRecord::Directory { order_key } => Some(&order_key.logical_path), + CanonicalExportRecord::File { order_key, .. } => Some(&order_key.logical_path), + }; + if let Some(logical_path) = logical_path { + if !logical_paths.insert(logical_path.clone()) { + return Err(ScopeContractError::DuplicateValue("logical_path")); + } + } + match record { + CanonicalExportRecord::Directory { order_key } => { + if logical_parent(&order_key.logical_path)? + .as_ref() + .is_some_and(|parent| { + !parent.as_str().eq_ignore_ascii_case(".loc") + && !directory_paths.contains(parent) + }) + { + return Err(ScopeContractError::MissingParentDirectory); + } + directory_paths.insert(order_key.logical_path.clone()); + } + CanonicalExportRecord::File { order_key, .. } => { + if order_key.parent_path.as_ref().is_some_and(|parent| { + !parent.as_str().eq_ignore_ascii_case(".loc") + && !directory_paths.contains(parent) + }) { + return Err(ScopeContractError::MissingParentDirectory); + } + } + CanonicalExportRecord::Control { .. } => {} + } + let key = record.order_key(); + if previous_key + .as_ref() + .is_some_and(|previous| previous >= &key) + { + return Err(ScopeContractError::NonCanonicalRecordOrder); + } + previous_key = Some(key); + } + if control_count != 1 { + return Err(ScopeContractError::InvalidControlRecordCount { + actual: control_count, + }); + } + Ok(()) +} + +/// Exact domain-separated, length-framed directory/file inventory preimage +/// shared by exporters and clients. The final control member is excluded to +/// avoid a receipt-hash recursion, and file bodies are never read here. +pub fn canonical_export_inventory_preimage( + records: &[CanonicalExportRecord], +) -> Result, ScopeContractError> { + validate_canonical_export_records(records)?; + let mut output = b"locality.export.inventory.v2\0".to_vec(); + append_count( + &mut output, + records + .iter() + .filter(|record| !matches!(record, CanonicalExportRecord::Control { .. })) + .count(), + )?; + for record in records { + match record { + CanonicalExportRecord::Control { .. } => {} + CanonicalExportRecord::Directory { order_key } => { + append_text(&mut output, "directory")?; + append_text(&mut output, order_key.logical_path.as_str())?; + } + CanonicalExportRecord::File { + order_key, + source_connection_id, + file_kind, + effective_actions, + content_sha256, + byte_length, + } => { + append_text(&mut output, "file")?; + append_u64(&mut output, u64::from(order_key.winning_scope_ordinal))?; + append_text(&mut output, source_connection_id.as_str())?; + append_text(&mut output, order_key.projection_id.as_str())?; + append_text(&mut output, order_key.logical_path.as_str())?; + append_text(&mut output, projection_file_kind_label(file_kind))?; + append_count(&mut output, effective_actions.len())?; + let mut action_labels = effective_actions + .iter() + .map(source_action_label) + .collect::>(); + action_labels.sort_unstable(); + for action_label in action_labels { + append_text(&mut output, action_label)?; + } + append_text(&mut output, content_sha256)?; + append_u64(&mut output, *byte_length)?; + } + } + } + Ok(output) +} + +pub fn canonical_export_inventory_sha256( + records: &[CanonicalExportRecord], +) -> Result { + let preimage = canonical_export_inventory_preimage(records)?; + Ok(format!("sha256:{:x}", Sha256::digest(preimage))) +} + +/// Incremental digest of delivered file bodies in canonical file order. +/// +/// The preimage begins with `locality.export.delivered-bodies.v2\0` and a u64 +/// big-endian file count. Each file contributes a length-framed projection ID +/// followed by its length-framed exact body bytes. Directory and final control +/// members are excluded. The sealed inventory binds each projection ID to its +/// path, metadata hash, and expected byte length. +pub struct DeliveredBodyDigestV2 { + hasher: Sha256, + expected_file_count: u64, + delivered_file_count: u64, +} + +impl DeliveredBodyDigestV2 { + pub fn new(expected_file_count: u64) -> Self { + let mut hasher = Sha256::new(); + hasher.update(b"locality.export.delivered-bodies.v2\0"); + hasher.update(expected_file_count.to_be_bytes()); + Self { + hasher, + expected_file_count, + delivered_file_count: 0, + } + } + + pub fn update_file( + &mut self, + projection_id: &ProjectionId, + body: &[u8], + ) -> Result<(), ScopeContractError> { + validate_nonempty("projection_id", projection_id.as_str())?; + update_digest_scalar(&mut self.hasher, projection_id.as_str().as_bytes())?; + update_digest_scalar(&mut self.hasher, body)?; + self.delivered_file_count = self + .delivered_file_count + .checked_add(1) + .ok_or(ScopeContractError::InventoryTooLarge)?; + Ok(()) + } + + pub fn finish(self) -> Result { + if self.delivered_file_count != self.expected_file_count { + return Err(ScopeContractError::DeliveredBodyCountMismatch { + expected: self.expected_file_count, + actual: self.delivered_file_count, + }); + } + Ok(format!("sha256:{:x}", self.hasher.finalize())) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct DeliveredCount { pub selected_entries: u64, @@ -394,6 +957,204 @@ pub enum TarContentEncoding { Zstd, } +/// Starts (or idempotently replays) export-attempt preflight. The idempotency +/// key is distinct from the server-assigned [`ExportAttemptId`]. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExportAttemptRequest { + pub versions: ComponentVersions, + pub opaque_session_capability: String, + pub idempotency_key: String, + pub content_encoding: TarContentEncoding, + pub limits: ExportAttemptLimits, +} + +impl Debug for ExportAttemptRequest { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ExportAttemptRequest") + .field("versions", &self.versions) + .field("opaque_session_capability", &"") + .field("idempotency_key", &"") + .field("content_encoding", &self.content_encoding) + .field("limits", &self.limits) + .finish() + } +} + +impl ExportAttemptRequest { + pub const MAX_IDEMPOTENCY_KEY_BYTES: usize = 128; + + pub fn validate(&self) -> Result<(), ScopeContractError> { + validate_export_versions(&self.versions)?; + validate_nonempty("opaque_session_capability", &self.opaque_session_capability)?; + validate_nonempty("idempotency_key", &self.idempotency_key)?; + if self.idempotency_key.len() > Self::MAX_IDEMPOTENCY_KEY_BYTES { + return Err(ScopeContractError::ValueTooLong { + field: "idempotency_key", + maximum_bytes: Self::MAX_IDEMPOTENCY_KEY_BYTES, + actual_bytes: self.idempotency_key.len(), + }); + } + self.limits.validate()?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExportAttemptLimits { + pub max_files: u64, + pub max_directories: u64, + pub max_content_bytes: u64, +} + +impl ExportAttemptLimits { + pub fn validate(&self) -> Result<(), ScopeContractError> { + if self.max_files == 0 || self.max_directories == 0 || self.max_content_bytes == 0 { + return Err(ScopeContractError::InvalidLimit); + } + Ok(()) + } +} + +/// Exact immutable selection sealed to one export attempt and source-head +/// generation vector before the first body byte is authorized. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SealedExportOffer { + pub versions: ComponentVersions, + pub session_id: SessionId, + pub export_attempt_id: ExportAttemptId, + pub source_generations: Vec, + pub media_type: String, + pub content_encoding: TarContentEncoding, + pub limits: ExportAttemptLimits, + pub control_entry_count: u64, + pub file_count: u64, + pub directory_count: u64, + pub archive_entry_count: u64, + pub selected_content_bytes: u64, + pub inventory_sha256: String, + pub sealed_at: String, + pub expires_at: String, +} + +impl SealedExportOffer { + pub fn validate(&self) -> Result<(), ScopeContractError> { + validate_export_versions(&self.versions)?; + validate_nonempty("session_id", self.session_id.as_str())?; + validate_source_generations(&self.source_generations)?; + validate_nonempty("media_type", &self.media_type)?; + validate_nonempty("sealed_at", &self.sealed_at)?; + validate_nonempty("expires_at", &self.expires_at)?; + self.limits.validate()?; + validate_export_counts( + self.control_entry_count, + self.file_count, + self.directory_count, + self.archive_entry_count, + )?; + validate_selection_limits( + &self.limits, + self.file_count, + self.directory_count, + self.selected_content_bytes, + )?; + validate_sha256("inventory_sha256", &self.inventory_sha256)?; + Ok(()) + } + + pub fn validate_inventory( + &self, + records: &[CanonicalExportRecord], + ) -> Result<(), ScopeContractError> { + self.validate()?; + validate_canonical_export_records(records)?; + let mut control_entry_count = 0_u64; + let mut directory_count = 0_u64; + let mut file_count = 0_u64; + let mut selected_content_bytes = 0_u64; + for record in records { + match record { + CanonicalExportRecord::Control { .. } => control_entry_count += 1, + CanonicalExportRecord::Directory { .. } => directory_count += 1, + CanonicalExportRecord::File { byte_length, .. } => { + file_count += 1; + selected_content_bytes = selected_content_bytes + .checked_add(*byte_length) + .ok_or(ScopeContractError::InventoryTooLarge)?; + } + } + } + let inventory_sha256 = canonical_export_inventory_sha256(records)?; + let archive_entry_count = + u64::try_from(records.len()).map_err(|_| ScopeContractError::InventoryTooLarge)?; + if control_entry_count != self.control_entry_count + || directory_count != self.directory_count + || file_count != self.file_count + || archive_entry_count != self.archive_entry_count + || selected_content_bytes != self.selected_content_bytes + || inventory_sha256 != self.inventory_sha256 + { + return Err(ScopeContractError::InventoryDoesNotMatchOffer); + } + Ok(()) + } +} + +/// Durable completion facts embedded in the final reserved control member. +/// Clean tar/Zstd EOF, this unique final receipt, and independently recomputed +/// inventory equality together gate atomic tree publication. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExportCompletionReceipt { + pub versions: ComponentVersions, + pub session_id: SessionId, + pub export_attempt_id: ExportAttemptId, + pub source_generations: Vec, + pub inventory_sha256: String, + pub delivered_control_entry_count: u64, + pub delivered_file_count: u64, + pub delivered_directory_count: u64, + pub delivered_archive_entry_count: u64, + pub delivered_content_bytes: u64, + pub delivered_body_sha256: String, + pub completed_at: String, +} + +impl ExportCompletionReceipt { + pub fn validate(&self) -> Result<(), ScopeContractError> { + validate_export_versions(&self.versions)?; + validate_nonempty("session_id", self.session_id.as_str())?; + validate_source_generations(&self.source_generations)?; + validate_nonempty("completed_at", &self.completed_at)?; + validate_export_counts( + self.delivered_control_entry_count, + self.delivered_file_count, + self.delivered_directory_count, + self.delivered_archive_entry_count, + )?; + validate_sha256("inventory_sha256", &self.inventory_sha256)?; + validate_sha256("delivered_body_sha256", &self.delivered_body_sha256)?; + Ok(()) + } + + pub fn validate_against(&self, offer: &SealedExportOffer) -> Result<(), ScopeContractError> { + self.validate()?; + offer.validate()?; + if self.session_id != offer.session_id + || self.export_attempt_id != offer.export_attempt_id + || self.source_generations != offer.source_generations + || self.inventory_sha256 != offer.inventory_sha256 + || self.delivered_control_entry_count != offer.control_entry_count + || self.delivered_file_count != offer.file_count + || self.delivered_directory_count != offer.directory_count + || self.delivered_archive_entry_count != offer.archive_entry_count + || self.delivered_content_bytes != offer.selected_content_bytes + { + return Err(ScopeContractError::ReceiptDoesNotMatchOffer); + } + Ok(()) + } +} + /// Encodings and exact decoded bounds available for one immutable tar export. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct TarExportOffer { @@ -692,6 +1453,292 @@ pub struct ChangesetStatusDetail { pub retriable: bool, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ScopeContractError { + VersionCompatibility(VersionCompatibilityError), + ComponentVersionTooOld { + component: ProtocolComponent, + required: u16, + actual: u16, + }, + UnsupportedSelectorVersion { + version: u16, + }, + EmptyField(&'static str), + EmptyCollection(&'static str), + DuplicateValue(&'static str), + NonCanonicalOrdinal { + collection: &'static str, + expected: u32, + actual: u32, + }, + InvalidLimit, + ValueTooLong { + field: &'static str, + maximum_bytes: usize, + actual_bytes: usize, + }, + InvalidSha256(&'static str), + InconsistentArchiveEntryCount, + SelectionExceedsLimits, + InvalidControlRecord, + InvalidControlDirectory, + InvalidControlRecordCount { + actual: u64, + }, + NonCanonicalRecordOrder, + InvalidDirectoryDepth { + expected: u32, + actual: u32, + }, + InvalidParentPath, + MissingParentDirectory, + ReceiptDoesNotMatchOffer, + InventoryTooLarge, + InventoryDoesNotMatchOffer, + DeliveredBodyCountMismatch { + expected: u64, + actual: u64, + }, +} + +impl From for ScopeContractError { + fn from(error: VersionCompatibilityError) -> Self { + Self::VersionCompatibility(error) + } +} + +impl Display for ScopeContractError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::VersionCompatibility(error) => Display::fmt(error, formatter), + Self::ComponentVersionTooOld { + component, + required, + actual, + } => write!( + formatter, + "{component:?} version {actual} is older than required version {required}" + ), + Self::UnsupportedSelectorVersion { version } => { + write!( + formatter, + "source-scope selector version {version} is unsupported" + ) + } + Self::EmptyField(field) => write!(formatter, "{field} must not be empty"), + Self::EmptyCollection(field) => write!(formatter, "{field} must not be empty"), + Self::DuplicateValue(field) => write!(formatter, "{field} must be unique"), + Self::NonCanonicalOrdinal { + collection, + expected, + actual, + } => write!( + formatter, + "{collection} ordinal must be {expected}, got {actual}" + ), + Self::InvalidLimit => formatter.write_str("export limits must be positive"), + Self::ValueTooLong { + field, + maximum_bytes, + actual_bytes, + } => write!( + formatter, + "{field} is {actual_bytes} bytes, exceeding {maximum_bytes} bytes" + ), + Self::InvalidSha256(field) => { + write!( + formatter, + "{field} must be `sha256:` plus 64 lowercase hex digits" + ) + } + Self::InconsistentArchiveEntryCount => formatter.write_str( + "archive_entry_count must equal one control entry plus directories and files", + ), + Self::SelectionExceedsLimits => { + formatter.write_str("sealed export selection exceeds negotiated limits") + } + Self::InvalidControlRecord => formatter.write_str( + "control record must be ordinal zero at the reserved export metadata path", + ), + Self::InvalidControlDirectory => { + formatter.write_str("the implicit .loc control directory must not be emitted") + } + Self::InvalidControlRecordCount { actual } => { + write!( + formatter, + "canonical export must contain one control record, got {actual}" + ) + } + Self::NonCanonicalRecordOrder => { + formatter.write_str("canonical export records are not in strict canonical order") + } + Self::InvalidDirectoryDepth { expected, actual } => write!( + formatter, + "directory depth must be {expected}, got {actual}" + ), + Self::InvalidParentPath => { + formatter.write_str("file parent_path does not match logical_path") + } + Self::MissingParentDirectory => { + formatter.write_str("canonical export record is missing its parent directory") + } + Self::ReceiptDoesNotMatchOffer => { + formatter.write_str("completion receipt does not match sealed export offer") + } + Self::InventoryTooLarge => { + formatter.write_str("canonical export inventory exceeds u64 framing") + } + Self::InventoryDoesNotMatchOffer => { + formatter.write_str("canonical export inventory does not match sealed offer") + } + Self::DeliveredBodyCountMismatch { expected, actual } => write!( + formatter, + "delivered body count must be {expected}, got {actual}" + ), + } + } +} + +impl std::error::Error for ScopeContractError {} + +fn validate_nonempty(field: &'static str, value: &str) -> Result<(), ScopeContractError> { + if value.is_empty() { + return Err(ScopeContractError::EmptyField(field)); + } + Ok(()) +} + +fn append_count(output: &mut Vec, count: usize) -> Result<(), ScopeContractError> { + let count = u64::try_from(count).map_err(|_| ScopeContractError::InventoryTooLarge)?; + output.extend_from_slice(&count.to_be_bytes()); + Ok(()) +} + +fn update_digest_scalar(hasher: &mut Sha256, value: &[u8]) -> Result<(), ScopeContractError> { + let length = u64::try_from(value.len()).map_err(|_| ScopeContractError::InventoryTooLarge)?; + hasher.update(length.to_be_bytes()); + hasher.update(value); + Ok(()) +} + +fn append_scalar(output: &mut Vec, value: &[u8]) -> Result<(), ScopeContractError> { + append_count(output, value.len())?; + output.extend_from_slice(value); + Ok(()) +} + +fn append_text(output: &mut Vec, value: &str) -> Result<(), ScopeContractError> { + append_scalar(output, value.as_bytes()) +} + +fn append_u64(output: &mut Vec, value: u64) -> Result<(), ScopeContractError> { + append_scalar(output, &value.to_be_bytes()) +} + +fn projection_file_kind_label(kind: &ProjectionFileKind) -> &'static str { + match kind { + ProjectionFileKind::Markdown => "markdown", + ProjectionFileKind::Text => "text", + ProjectionFileKind::Json => "json", + ProjectionFileKind::Yaml => "yaml", + ProjectionFileKind::Binary => "binary", + ProjectionFileKind::Directory => "directory", + } +} + +fn source_action_label(action: &SourceAction) -> &'static str { + match action { + SourceAction::Read => "read", + SourceAction::Search => "search", + SourceAction::DownloadAttachment => "download_attachment", + SourceAction::Create => "create", + SourceAction::Update => "update", + SourceAction::Move => "move", + SourceAction::Delete => "delete", + SourceAction::Comment => "comment", + SourceAction::UpdateProperties => "update_properties", + SourceAction::ManageSchema => "manage_schema", + } +} + +fn validate_sha256(field: &'static str, value: &str) -> Result<(), ScopeContractError> { + let Some(hex) = value.strip_prefix("sha256:") else { + return Err(ScopeContractError::InvalidSha256(field)); + }; + if hex.len() != 64 + || !hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(ScopeContractError::InvalidSha256(field)); + } + Ok(()) +} + +fn validate_export_versions(versions: &ComponentVersions) -> Result<(), ScopeContractError> { + versions.validate_required()?; + for (component, actual) in [ + (ProtocolComponent::Session, versions.session), + (ProtocolComponent::Replica, versions.replica), + (ProtocolComponent::ExportMetadata, versions.export_metadata), + ] { + if actual < 2 { + return Err(ScopeContractError::ComponentVersionTooOld { + component, + required: 2, + actual, + }); + } + } + Ok(()) +} + +fn validate_export_counts( + control_entry_count: u64, + file_count: u64, + directory_count: u64, + archive_entry_count: u64, +) -> Result<(), ScopeContractError> { + if control_entry_count != 1 + || control_entry_count + .checked_add(directory_count) + .and_then(|count| count.checked_add(file_count)) + != Some(archive_entry_count) + { + return Err(ScopeContractError::InconsistentArchiveEntryCount); + } + Ok(()) +} + +fn validate_selection_limits( + limits: &ExportAttemptLimits, + file_count: u64, + directory_count: u64, + selected_content_bytes: u64, +) -> Result<(), ScopeContractError> { + if file_count > limits.max_files + || directory_count > limits.max_directories + || selected_content_bytes > limits.max_content_bytes + { + return Err(ScopeContractError::SelectionExceedsLimits); + } + Ok(()) +} + +fn path_depth(path: &LogicalPath) -> u32 { + path.as_str().split('/').count() as u32 +} + +fn logical_parent(path: &LogicalPath) -> Result, ScopeContractError> { + path.as_str() + .rsplit_once('/') + .map(|(parent, _)| { + LogicalPath::new(parent).map_err(|_| ScopeContractError::InvalidParentPath) + }) + .transpose() +} + pub const COMPONENT_VERSIONS_GOLDEN_JSON: &[u8] = include_bytes!("../fixtures/component-versions.json"); pub const AUTHORIZED_SESSION_QUERY_GOLDEN_JSON: &[u8] = @@ -720,3 +1767,15 @@ pub const SESSION_PROTOCOL_ERROR_GOLDEN_JSON: &[u8] = pub const TAR_EXPORT_OFFER_GOLDEN_JSON: &[u8] = include_bytes!("../fixtures/tar-export-offer.json"); pub const TAR_EXPORT_METADATA_GOLDEN_JSON: &[u8] = include_bytes!("../fixtures/tar-export-metadata.json"); +pub const SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON: &[u8] = + include_bytes!("../fixtures/scope-authorized-session-query.json"); +pub const EXPORT_ATTEMPT_REQUEST_GOLDEN_JSON: &[u8] = + include_bytes!("../fixtures/export-attempt-request.json"); +pub const SEALED_EXPORT_OFFER_GOLDEN_JSON: &[u8] = + include_bytes!("../fixtures/sealed-export-offer.json"); +pub const EXPORT_COMPLETION_RECEIPT_GOLDEN_JSON: &[u8] = + include_bytes!("../fixtures/export-completion-receipt.json"); +pub const CANONICAL_EXPORT_RECORDS_GOLDEN_JSON: &[u8] = + include_bytes!("../fixtures/canonical-export-records.json"); +pub const CANONICAL_EXPORT_INVENTORY_GOLDEN_JSON: &[u8] = + include_bytes!("../fixtures/canonical-export-inventory.json"); diff --git a/crates/locality-protocol/tests/golden.rs b/crates/locality-protocol/tests/golden.rs index eb8a51a6..ff4af572 100644 --- a/crates/locality-protocol/tests/golden.rs +++ b/crates/locality-protocol/tests/golden.rs @@ -4,32 +4,41 @@ use locality_core::journal::PushOperationId; use locality_core::model::RemoteId; use locality_core::planner::PlanSummary; use locality_core::portable::{ - AccessSetId, ChangesetId, ContentVersionId, LogicalPath, PrincipalId, ProjectionEntry, - ProjectionFileKind, ProjectionId, ProjectionInput, ProjectionVersionId, ReplicaRevisionId, - SessionId, SourceAction, SourceConnectionId, SourceOperation, SourceOperationPlan, - SourceVersionId, TenantId, + AccessSetId, ChangesetId, ContentVersionId, ExportAttemptId, LogicalPath, PrincipalId, + ProjectionEntry, ProjectionFileKind, ProjectionId, ProjectionInput, ProjectionVersionId, + ReplicaRevisionId, SessionId, SourceAction, SourceConnectionId, SourceGenerationId, + SourceOperation, SourceOperationPlan, SourceScopeId, SourceVersionId, TenantId, }; use locality_core::readable_diff::readable_diff_for_file; use locality_protocol::{ ACCESS_SET_GOLDEN_JSON, AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, AccessSetContract, AccessSubject, - AuditReference, AuthorizedChangesetUpload, AuthorizedSessionQuery, - BOOTSTRAP_EXCHANGE_GOLDEN_JSON, BootstrapExchangeRequest, CHANGESET_ENVELOPE_GOLDEN_JSON, - COMPONENT_VERSIONS, COMPONENT_VERSIONS_GOLDEN_JSON, CONTENT_VERSION_GOLDEN_JSON, - ChangesetContent, ChangesetEnvelope, ChangesetSourceObject, ClientValidationResult, - ComponentVersions, ContentVersionContract, DELIVERED_COUNT_GOLDEN_JSON, DeliveredChangesetBase, - DeliveredCount, EditedCanonicalBody, FRESHNESS_STATUS_GOLDEN_JSON, FreshnessRequirement, - ORDERED_EXPORT_ROWS_GOLDEN_JSON, OpaqueBootstrapExchangeRequest, OpaqueSessionStatusRequest, - OrderedExportRow, PROJECTION_VERSION_GOLDEN_JSON, ProjectionVersionContract, - READY_REPLICA_REVISION_GOLDEN_JSON, ReadyReplicaRevision, ReplicaFreshnessState, - ReplicaFreshnessStatus, SANDBOX_SESSION_STATUS_GOLDEN_JSON, SESSION_PROTOCOL_ERROR_GOLDEN_JSON, - SOURCE_VERSION_GOLDEN_JSON, SandboxSessionState, SandboxSessionStatus, SessionCapability, + AuditReference, AuthorizedChangesetUpload, AuthorizedSessionQuery, AuthorizedSourceScope, + BOOTSTRAP_EXCHANGE_GOLDEN_JSON, BootstrapExchangeRequest, + CANONICAL_EXPORT_INVENTORY_GOLDEN_JSON, CANONICAL_EXPORT_RECORDS_GOLDEN_JSON, + CHANGESET_ENVELOPE_GOLDEN_JSON, COMPONENT_VERSIONS, COMPONENT_VERSIONS_GOLDEN_JSON, + CONTENT_VERSION_GOLDEN_JSON, CanonicalControlOrderKey, CanonicalDirectoryOrderKey, + CanonicalExportRecord, CanonicalFileOrderKey, ChangesetContent, ChangesetEnvelope, + ChangesetSourceObject, ClientValidationResult, CompatibleAuthorizedSessionQuery, + ComponentVersions, ContentVersionContract, DELIVERED_COUNT_GOLDEN_JSON, DeliveredBodyDigestV2, + DeliveredChangesetBase, DeliveredCount, EXPORT_ATTEMPT_REQUEST_GOLDEN_JSON, + EXPORT_COMPLETION_RECEIPT_GOLDEN_JSON, EditedCanonicalBody, ExportAttemptLimits, + ExportAttemptRequest, ExportCompletionReceipt, FRESHNESS_STATUS_GOLDEN_JSON, + FreshnessRequirement, NotionScopeKind, ORDERED_EXPORT_ROWS_GOLDEN_JSON, + OpaqueBootstrapExchangeRequest, OpaqueSessionStatusRequest, OrderedExportRow, + OrderedSourceGeneration, PROJECTION_VERSION_GOLDEN_JSON, ProjectionVersionContract, + ProviderSourceScopeSelector, READY_REPLICA_REVISION_GOLDEN_JSON, ReadyReplicaRevision, + ReplicaFreshnessState, ReplicaFreshnessStatus, SANDBOX_SESSION_STATUS_GOLDEN_JSON, + SCOPE_AUTHORIZED_COMPONENT_VERSIONS, SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, + SEALED_EXPORT_OFFER_GOLDEN_JSON, SESSION_PROTOCOL_ERROR_GOLDEN_JSON, + SOURCE_VERSION_GOLDEN_JSON, SandboxSessionState, SandboxSessionStatus, + ScopeAuthorizedSessionQuery, ScopeContractError, SealedExportOffer, SessionCapability, SessionErrorCode, SessionProtocolError, SessionReplicaRevision, SourceVersionContract, StaleSessionBehavior, TAR_EXPORT_METADATA_GOLDEN_JSON, TAR_EXPORT_OFFER_GOLDEN_JSON, TarContentEncoding, TarExportMetadata, TarExportOffer, WRITABLE_EXPORT_METADATA_GOLDEN_JSON, - WritableExportMetadata, WritableMetadataEntry, + WritableExportMetadata, WritableMetadataEntry, validate_canonical_export_records, }; -use serde::Serialize; use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; fn exact_pretty_json(value: &impl Serialize) -> Vec { let mut bytes = serde_json::to_vec_pretty(value).expect("serialize fixture"); @@ -46,6 +55,12 @@ where assert_eq!(exact_pretty_json(&decoded), golden); } +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +struct CanonicalInventoryGolden { + preimage_hex: String, + sha256: String, +} + #[test] fn component_versions_are_exact_golden_bytes() { assert_exact_round_trip(COMPONENT_VERSIONS_GOLDEN_JSON, &COMPONENT_VERSIONS); @@ -56,6 +71,137 @@ fn authorized_session_query_is_exact_golden_bytes() { assert_exact_round_trip(AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, &authorized_query()); } +#[test] +fn scope_authorized_export_contracts_are_exact_golden_bytes() { + let query = scope_authorized_query(); + query.validate().expect("valid scope query"); + assert_exact_round_trip(SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, &query); + + let request = export_attempt_request(); + request.validate().expect("valid attempt request"); + assert_exact_round_trip(EXPORT_ATTEMPT_REQUEST_GOLDEN_JSON, &request); + + let offer = sealed_export_offer(); + offer.validate().expect("valid sealed offer"); + assert_exact_round_trip(SEALED_EXPORT_OFFER_GOLDEN_JSON, &offer); + + let receipt = export_completion_receipt(); + receipt + .validate_against(&offer) + .expect("completion matches sealed metadata selection"); + assert_exact_round_trip(EXPORT_COMPLETION_RECEIPT_GOLDEN_JSON, &receipt); + + let records = canonical_export_records(); + validate_canonical_export_records(&records).expect("canonical record order"); + let preimage = locality_protocol::canonical_export_inventory_preimage(&records) + .expect("inventory preimage"); + let inventory = CanonicalInventoryGolden { + preimage_hex: preimage.iter().map(|byte| format!("{byte:02x}")).collect(), + sha256: locality_protocol::canonical_export_inventory_sha256(&records) + .expect("inventory digest"), + }; + offer + .validate_inventory(&records) + .expect("sealed offer matches exact canonical inventory"); + assert_exact_round_trip(CANONICAL_EXPORT_INVENTORY_GOLDEN_JSON, &inventory); + assert_exact_round_trip(CANONICAL_EXPORT_RECORDS_GOLDEN_JSON, &records); +} + +#[test] +fn compatibility_decoder_accepts_legacy_and_scope_queries() { + let legacy = serde_json::from_slice::( + AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, + ) + .expect("legacy query decodes"); + assert_eq!( + legacy, + CompatibleAuthorizedSessionQuery::Legacy(authorized_query()) + ); + + let scope = serde_json::from_slice::( + SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, + ) + .expect("scope query decodes"); + assert_eq!( + scope, + CompatibleAuthorizedSessionQuery::Scope(scope_authorized_query()) + ); +} + +#[test] +fn scope_contract_validation_rejects_ambiguous_or_inexact_values() { + assert!(SourceScopeId::new("").is_err()); + assert!(ExportAttemptId::new("").is_err()); + assert!(SourceGenerationId::new("").is_err()); + assert!(serde_json::from_str::("\"\"").is_err()); + + let mut query = scope_authorized_query(); + query.authorized_scopes[1].ordinal = 0; + assert!(matches!( + query.validate(), + Err(ScopeContractError::NonCanonicalOrdinal { + collection: "authorized_scopes", + .. + }) + )); + + let mut offer = sealed_export_offer(); + offer.archive_entry_count -= 1; + assert_eq!( + offer.validate(), + Err(ScopeContractError::InconsistentArchiveEntryCount) + ); + + let mut duplicate_generation_offer = sealed_export_offer(); + duplicate_generation_offer + .source_generations + .push(OrderedSourceGeneration { + ordinal: 1, + source_connection_id: SourceConnectionId::new("source-notion"), + source_generation_id: SourceGenerationId::new("generation-notion-110") + .expect("generation id"), + }); + assert_eq!( + duplicate_generation_offer.validate(), + Err(ScopeContractError::DuplicateValue("source_connection_id")) + ); + + let mut receipt = export_completion_receipt(); + receipt.inventory_sha256 = format!("sha256:{}", "A".repeat(64)); + assert_eq!( + receipt.validate(), + Err(ScopeContractError::InvalidSha256("inventory_sha256")) + ); + + let mut request = export_attempt_request(); + request.idempotency_key = "x".repeat(ExportAttemptRequest::MAX_IDEMPOTENCY_KEY_BYTES + 1); + assert!(matches!( + request.validate(), + Err(ScopeContractError::ValueTooLong { + field: "idempotency_key", + .. + }) + )); + + let mut records = canonical_export_records(); + records.pop(); + assert_eq!( + validate_canonical_export_records(&records), + Err(ScopeContractError::InvalidControlRecordCount { actual: 0 }) + ); + + let implicit_control_directory = CanonicalExportRecord::Directory { + order_key: CanonicalDirectoryOrderKey { + depth: 1, + logical_path: LogicalPath::new(".loc").expect("path"), + }, + }; + assert_eq!( + implicit_control_directory.validate(), + Err(ScopeContractError::InvalidControlDirectory) + ); +} + #[test] fn source_content_projection_and_access_contracts_are_exact_golden_bytes() { assert_exact_round_trip(SOURCE_VERSION_GOLDEN_JSON, &source_version()); @@ -137,6 +283,11 @@ fn capability_debug_output_is_redacted() { let token_only_bootstrap = OpaqueBootstrapExchangeRequest { bootstrap_token: "token-only-secret".to_string(), }; + let export_attempt = ExportAttemptRequest { + opaque_session_capability: "attempt-capability-secret".to_string(), + idempotency_key: "attempt-idempotency-secret".to_string(), + ..export_attempt_request() + }; for (debug, secret) in [ (format!("{bootstrap:?}"), "bootstrap-secret"), @@ -144,6 +295,8 @@ fn capability_debug_output_is_redacted() { (format!("{upload:?}"), "upload-secret"), (format!("{status:?}"), "status-secret"), (format!("{token_only_bootstrap:?}"), "token-only-secret"), + (format!("{export_attempt:?}"), "attempt-capability-secret"), + (format!("{export_attempt:?}"), "attempt-idempotency-secret"), ] { assert!(debug.contains(""), "{debug}"); assert!(!debug.contains(secret), "{debug}"); @@ -169,6 +322,12 @@ fn public_goldens_never_expose_physical_serving_identifiers() { SESSION_PROTOCOL_ERROR_GOLDEN_JSON, TAR_EXPORT_OFFER_GOLDEN_JSON, TAR_EXPORT_METADATA_GOLDEN_JSON, + SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, + EXPORT_ATTEMPT_REQUEST_GOLDEN_JSON, + SEALED_EXPORT_OFFER_GOLDEN_JSON, + EXPORT_COMPLETION_RECEIPT_GOLDEN_JSON, + CANONICAL_EXPORT_RECORDS_GOLDEN_JSON, + CANONICAL_EXPORT_INVENTORY_GOLDEN_JSON, ] { let value: serde_json::Value = serde_json::from_slice(golden).expect("valid golden JSON"); assert_forbidden_physical_keys_absent(&value); @@ -296,6 +455,171 @@ fn authorized_query() -> AuthorizedSessionQuery { } } +fn scope_authorized_query() -> ScopeAuthorizedSessionQuery { + ScopeAuthorizedSessionQuery { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + tenant_id: TenantId::new("tenant-acme"), + session_id: SessionId::new("session-scope-7"), + acting_principal_id: PrincipalId::new("principal-agent"), + workload_id: "workload-sandbox".to_string(), + authorization_revision: 43, + policy_revision: 18, + profile_revision: 10, + authorized_scopes: vec![ + AuthorizedSourceScope { + ordinal: 0, + source_scope_id: SourceScopeId::new("scope-product").expect("scope id"), + source_connection_id: SourceConnectionId::new("source-notion"), + selector: ProviderSourceScopeSelector::Notion { + selector_version: 1, + scope_kind: NotionScopeKind::Page, + provider_scope_id: "e07dd2a2531444aba2f452010314fb87".to_string(), + }, + effective_actions: BTreeSet::from([ + SourceAction::Read, + SourceAction::DownloadAttachment, + ]), + validated_filter_digest: Some(format!("sha256:{}", "1".repeat(64))), + }, + AuthorizedSourceScope { + ordinal: 1, + source_scope_id: SourceScopeId::new("scope-go-to-market").expect("scope id"), + source_connection_id: SourceConnectionId::new("source-notion"), + selector: ProviderSourceScopeSelector::Notion { + selector_version: 1, + scope_kind: NotionScopeKind::Page, + provider_scope_id: "f525c3a7db684465a9e1d1082bea6f81".to_string(), + }, + effective_actions: BTreeSet::from([SourceAction::Read]), + validated_filter_digest: None, + }, + ], + max_files: 10_000, + max_directories: 10_000, + max_bytes: 104_857_600, + } +} + +fn attempt_limits() -> ExportAttemptLimits { + ExportAttemptLimits { + max_files: 10_000, + max_directories: 10_000, + max_content_bytes: 104_857_600, + } +} + +fn export_attempt_request() -> ExportAttemptRequest { + ExportAttemptRequest { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + opaque_session_capability: "opaque-session-capability".to_string(), + idempotency_key: "mount-attempt-2026-07-23T19:00:00Z".to_string(), + content_encoding: TarContentEncoding::Zstd, + limits: attempt_limits(), + } +} + +fn source_generations() -> Vec { + vec![OrderedSourceGeneration { + ordinal: 0, + source_connection_id: SourceConnectionId::new("source-notion"), + source_generation_id: SourceGenerationId::new("generation-notion-109") + .expect("generation id"), + }] +} + +fn sealed_export_offer() -> SealedExportOffer { + SealedExportOffer { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: SessionId::new("session-scope-7"), + export_attempt_id: ExportAttemptId::new("export-attempt-9").expect("attempt id"), + source_generations: source_generations(), + media_type: "application/x-tar".to_string(), + content_encoding: TarContentEncoding::Zstd, + limits: attempt_limits(), + control_entry_count: 1, + file_count: 2, + directory_count: 2, + archive_entry_count: 5, + selected_content_bytes: 17, + inventory_sha256: "sha256:025cdbae136931542f7fa881da423e8e1f29a6132cf26ae5f4eea53c53a8ef51" + .to_string(), + sealed_at: "2026-07-23T19:00:01Z".to_string(), + expires_at: "2026-07-23T19:10:01Z".to_string(), + } +} + +fn export_completion_receipt() -> ExportCompletionReceipt { + let mut body_digest = DeliveredBodyDigestV2::new(2); + body_digest + .update_file(&ProjectionId::new("projection-roadmap"), b"# Roadmap\n") + .expect("body digest"); + body_digest + .update_file(&ProjectionId::new("projection-readme"), b"Public\n") + .expect("body digest"); + ExportCompletionReceipt { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: SessionId::new("session-scope-7"), + export_attempt_id: ExportAttemptId::new("export-attempt-9").expect("attempt id"), + source_generations: source_generations(), + inventory_sha256: "sha256:025cdbae136931542f7fa881da423e8e1f29a6132cf26ae5f4eea53c53a8ef51" + .to_string(), + delivered_control_entry_count: 1, + delivered_file_count: 2, + delivered_directory_count: 2, + delivered_archive_entry_count: 5, + delivered_content_bytes: 17, + delivered_body_sha256: body_digest.finish().expect("body digest"), + completed_at: "2026-07-23T19:00:04Z".to_string(), + } +} + +fn canonical_export_records() -> Vec { + vec![ + CanonicalExportRecord::Directory { + order_key: CanonicalDirectoryOrderKey { + depth: 1, + logical_path: LogicalPath::new("Projects").expect("path"), + }, + }, + CanonicalExportRecord::Directory { + order_key: CanonicalDirectoryOrderKey { + depth: 2, + logical_path: LogicalPath::new("Projects/Roadmap").expect("path"), + }, + }, + CanonicalExportRecord::File { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: Some(LogicalPath::new("Projects/Roadmap").expect("path")), + logical_path: LogicalPath::new("Projects/Roadmap/page.md").expect("path"), + projection_id: ProjectionId::new("projection-roadmap"), + }, + source_connection_id: SourceConnectionId::new("source-notion"), + file_kind: ProjectionFileKind::Markdown, + effective_actions: BTreeSet::from([SourceAction::Read, SourceAction::Update]), + content_sha256: format!("sha256:{}", "5".repeat(64)), + byte_length: 10, + }, + CanonicalExportRecord::File { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: 1, + parent_path: None, + logical_path: LogicalPath::new("README.md").expect("path"), + projection_id: ProjectionId::new("projection-readme"), + }, + source_connection_id: SourceConnectionId::new("source-notion"), + file_kind: ProjectionFileKind::Markdown, + effective_actions: BTreeSet::from([SourceAction::Read]), + content_sha256: format!("sha256:{}", "6".repeat(64)), + byte_length: 7, + }, + CanonicalExportRecord::Control { + order_key: CanonicalControlOrderKey { ordinal: 0 }, + member_path: locality_protocol::RESERVED_EXPORT_METADATA_PATH.to_string(), + }, + ] +} + fn source_version() -> SourceVersionContract { SourceVersionContract { tenant_id: TenantId::new("tenant-acme"), From 899ad3209fca7f5473ca896fb706af2cb024afdd Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 15:29:39 -0700 Subject: [PATCH 04/14] Define export v2 archive metadata contract --- crates/locality-protocol/src/lib.rs | 51 ++++++++++++++++++++++-- crates/locality-protocol/tests/golden.rs | 49 +++++++++++++++++++++++ docs/cloud-sandbox-data-plane.md | 6 +++ 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/crates/locality-protocol/src/lib.rs b/crates/locality-protocol/src/lib.rs index 2b0e1df0..b0d5091b 100644 --- a/crates/locality-protocol/src/lib.rs +++ b/crates/locality-protocol/src/lib.rs @@ -21,6 +21,21 @@ use sha2::{Digest, Sha256}; pub use locality_core::portable::RESERVED_EXPORT_METADATA_PATH; +pub const PAX_SOURCE_CONNECTION_ID: &str = "locality.source_connection_id"; +pub const PAX_PROJECTION_ID: &str = "locality.projection_id"; +pub const PAX_WINNING_SCOPE_ORDINAL: &str = "locality.winning_scope_ordinal"; +pub const PAX_FILE_KIND: &str = "locality.file_kind"; +pub const PAX_EFFECTIVE_ACTIONS: &str = "locality.effective_actions"; +pub const PAX_CONTENT_SHA256: &str = "locality.content_sha256"; +pub const EXPORT_V2_FILE_PAX_KEYS: [&str; 6] = [ + PAX_SOURCE_CONNECTION_ID, + PAX_PROJECTION_ID, + PAX_WINNING_SCOPE_ORDINAL, + PAX_FILE_KIND, + PAX_EFFECTIVE_ACTIONS, + PAX_CONTENT_SHA256, +]; + pub const COMPONENT_VERSIONS: ComponentVersions = ComponentVersions { session: 1, replica: 1, @@ -753,11 +768,11 @@ pub fn canonical_export_inventory_preimage( append_text(&mut output, source_connection_id.as_str())?; append_text(&mut output, order_key.projection_id.as_str())?; append_text(&mut output, order_key.logical_path.as_str())?; - append_text(&mut output, projection_file_kind_label(file_kind))?; + append_text(&mut output, projection_file_kind_wire_label(file_kind))?; append_count(&mut output, effective_actions.len())?; let mut action_labels = effective_actions .iter() - .map(source_action_label) + .map(source_action_wire_label) .collect::>(); action_labels.sort_unstable(); for action_label in action_labels { @@ -1636,7 +1651,7 @@ fn append_u64(output: &mut Vec, value: u64) -> Result<(), ScopeContractError append_scalar(output, &value.to_be_bytes()) } -fn projection_file_kind_label(kind: &ProjectionFileKind) -> &'static str { +pub fn projection_file_kind_wire_label(kind: &ProjectionFileKind) -> &'static str { match kind { ProjectionFileKind::Markdown => "markdown", ProjectionFileKind::Text => "text", @@ -1647,7 +1662,19 @@ fn projection_file_kind_label(kind: &ProjectionFileKind) -> &'static str { } } -fn source_action_label(action: &SourceAction) -> &'static str { +pub fn projection_file_kind_from_wire_label(value: &str) -> Option { + match value { + "markdown" => Some(ProjectionFileKind::Markdown), + "text" => Some(ProjectionFileKind::Text), + "json" => Some(ProjectionFileKind::Json), + "yaml" => Some(ProjectionFileKind::Yaml), + "binary" => Some(ProjectionFileKind::Binary), + "directory" => Some(ProjectionFileKind::Directory), + _ => None, + } +} + +pub fn source_action_wire_label(action: &SourceAction) -> &'static str { match action { SourceAction::Read => "read", SourceAction::Search => "search", @@ -1662,6 +1689,22 @@ fn source_action_label(action: &SourceAction) -> &'static str { } } +pub fn source_action_from_wire_label(value: &str) -> Option { + match value { + "read" => Some(SourceAction::Read), + "search" => Some(SourceAction::Search), + "download_attachment" => Some(SourceAction::DownloadAttachment), + "create" => Some(SourceAction::Create), + "update" => Some(SourceAction::Update), + "move" => Some(SourceAction::Move), + "delete" => Some(SourceAction::Delete), + "comment" => Some(SourceAction::Comment), + "update_properties" => Some(SourceAction::UpdateProperties), + "manage_schema" => Some(SourceAction::ManageSchema), + _ => None, + } +} + fn validate_sha256(field: &'static str, value: &str) -> Result<(), ScopeContractError> { let Some(hex) = value.strip_prefix("sha256:") else { return Err(ScopeContractError::InvalidSha256(field)); diff --git a/crates/locality-protocol/tests/golden.rs b/crates/locality-protocol/tests/golden.rs index ff4af572..f445a8f3 100644 --- a/crates/locality-protocol/tests/golden.rs +++ b/crates/locality-protocol/tests/golden.rs @@ -303,6 +303,55 @@ fn capability_debug_output_is_redacted() { } } +#[test] +fn export_v2_pax_wire_contract_is_exact_and_round_trips() { + assert_eq!( + locality_protocol::EXPORT_V2_FILE_PAX_KEYS, + [ + "locality.source_connection_id", + "locality.projection_id", + "locality.winning_scope_ordinal", + "locality.file_kind", + "locality.effective_actions", + "locality.content_sha256", + ] + ); + + for kind in [ + ProjectionFileKind::Markdown, + ProjectionFileKind::Text, + ProjectionFileKind::Json, + ProjectionFileKind::Yaml, + ProjectionFileKind::Binary, + ProjectionFileKind::Directory, + ] { + let label = locality_protocol::projection_file_kind_wire_label(&kind); + assert_eq!( + locality_protocol::projection_file_kind_from_wire_label(label), + Some(kind) + ); + } + + for action in [ + SourceAction::Read, + SourceAction::Search, + SourceAction::DownloadAttachment, + SourceAction::Create, + SourceAction::Update, + SourceAction::Move, + SourceAction::Delete, + SourceAction::Comment, + SourceAction::UpdateProperties, + SourceAction::ManageSchema, + ] { + let label = locality_protocol::source_action_wire_label(&action); + assert_eq!( + locality_protocol::source_action_from_wire_label(label), + Some(action) + ); + } +} + #[test] fn public_goldens_never_expose_physical_serving_identifiers() { for golden in [ diff --git a/docs/cloud-sandbox-data-plane.md b/docs/cloud-sandbox-data-plane.md index e86828ce..3251b016 100644 --- a/docs/cloud-sandbox-data-plane.md +++ b/docs/cloud-sandbox-data-plane.md @@ -1322,6 +1322,12 @@ Content-Encoding: zstd path, size, and non-executable mode. File members follow `(winning_scope_ordinal, parent_path COLLATE "C", logical_path COLLATE "C", projection_id)` order; +- each v2 file member carries bounded transient POSIX PAX fields for source + connection ID, projection ID, winning scope ordinal, file kind, sorted + effective actions, and content SHA-256. Public constants and parsers define + their exact names/encoding. They allow the client to recompute the inventory + and body digests but are never materialized or inserted into SQLite; directory + and control members carry no Locality PAX fields; - v1 retains its reserved first writable-metadata member unchanged. V2 instead requires exactly one final record-class-2 `.loc/session.json` member combining that writable metadata with the completion receipt. It is bounded, canonical From 089d4853c0cc968d4c2b16f77dc070722a17dd5a Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 15:32:53 -0700 Subject: [PATCH 05/14] Canonicalize export action metadata --- crates/locality-protocol/src/lib.rs | 30 ++++++++++++++++++++++++ crates/locality-protocol/tests/golden.rs | 29 +++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/crates/locality-protocol/src/lib.rs b/crates/locality-protocol/src/lib.rs index b0d5091b..155a0f31 100644 --- a/crates/locality-protocol/src/lib.rs +++ b/crates/locality-protocol/src/lib.rs @@ -1705,6 +1705,36 @@ pub fn source_action_from_wire_label(value: &str) -> Option { } } +/// Encodes the effective actions carried by an export-v2 file PAX record. +/// +/// The wire value is a compact JSON array whose labels are sorted by their +/// UTF-8 bytes. An empty action set is invalid for an exported file. +pub fn canonical_effective_actions_pax_value(actions: &BTreeSet) -> Option { + if actions.is_empty() { + return None; + } + let mut labels = actions + .iter() + .map(source_action_wire_label) + .collect::>(); + labels.sort_unstable(); + Some(format!(r#"["{}"]"#, labels.join(r#"",""#))) +} + +/// Decodes an export-v2 effective-actions PAX value only when it is already in +/// the exact canonical encoding accepted by the inventory digest. +pub fn source_actions_from_canonical_pax_value(value: &str) -> Option> { + let labels = value.strip_prefix(r#"[""#)?.strip_suffix(r#""]"#)?; + let mut actions = BTreeSet::new(); + for label in labels.split(r#"",""#) { + let action = source_action_from_wire_label(label)?; + if !actions.insert(action) { + return None; + } + } + (canonical_effective_actions_pax_value(&actions)?.as_str() == value).then_some(actions) +} + fn validate_sha256(field: &'static str, value: &str) -> Result<(), ScopeContractError> { let Some(hex) = value.strip_prefix("sha256:") else { return Err(ScopeContractError::InvalidSha256(field)); diff --git a/crates/locality-protocol/tests/golden.rs b/crates/locality-protocol/tests/golden.rs index f445a8f3..ad3b9829 100644 --- a/crates/locality-protocol/tests/golden.rs +++ b/crates/locality-protocol/tests/golden.rs @@ -350,6 +350,35 @@ fn export_v2_pax_wire_contract_is_exact_and_round_trips() { Some(action) ); } + + let actions = BTreeSet::from([ + SourceAction::UpdateProperties, + SourceAction::Read, + SourceAction::DownloadAttachment, + ]); + let encoded = locality_protocol::canonical_effective_actions_pax_value(&actions).unwrap(); + assert_eq!( + encoded, + r#"["download_attachment","read","update_properties"]"# + ); + assert_eq!( + locality_protocol::source_actions_from_canonical_pax_value(&encoded), + Some(actions) + ); + for invalid in [ + "[]", + r#"["read","read"]"#, + r#"["read", "update"]"#, + r#"["update","read"]"#, + r#"["unknown"]"#, + r#"{"action":"read"}"#, + ] { + assert_eq!( + locality_protocol::source_actions_from_canonical_pax_value(invalid), + None, + "accepted non-canonical effective actions: {invalid}" + ); + } } #[test] From 98c9a9f2a3f316cc7e72b6dc853de0cc6cee07f8 Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 15:46:08 -0700 Subject: [PATCH 06/14] Seal export v2 terminal controls --- .../fixtures/export-completion-receipt.json | 1 + .../fixtures/export-terminal-control-v2.json | 67 ++++ .../fixtures/sealed-export-offer.json | 1 + crates/locality-protocol/src/lib.rs | 355 +++++++++++++++++- crates/locality-protocol/tests/golden.rs | 151 +++++++- docs/cloud-sandbox-data-plane.md | 25 +- 6 files changed, 570 insertions(+), 30 deletions(-) create mode 100644 crates/locality-protocol/fixtures/export-terminal-control-v2.json diff --git a/crates/locality-protocol/fixtures/export-completion-receipt.json b/crates/locality-protocol/fixtures/export-completion-receipt.json index 8e49b4d4..b2f0635a 100644 --- a/crates/locality-protocol/fixtures/export-completion-receipt.json +++ b/crates/locality-protocol/fixtures/export-completion-receipt.json @@ -18,6 +18,7 @@ } ], "inventory_sha256": "sha256:025cdbae136931542f7fa881da423e8e1f29a6132cf26ae5f4eea53c53a8ef51", + "writable_metadata_sha256": "sha256:a14c564c8d8950755b30115c4e8c684017662c70e7e03ead0d65a5da132474d2", "delivered_control_entry_count": 1, "delivered_file_count": 2, "delivered_directory_count": 2, diff --git a/crates/locality-protocol/fixtures/export-terminal-control-v2.json b/crates/locality-protocol/fixtures/export-terminal-control-v2.json new file mode 100644 index 00000000..188ed22b --- /dev/null +++ b/crates/locality-protocol/fixtures/export-terminal-control-v2.json @@ -0,0 +1,67 @@ +{ + "writable_metadata": { + "versions": { + "session": 2, + "replica": 2, + "export_metadata": 2, + "writable_session_store": 1, + "canonical": 1, + "path": 1, + "changeset": 1 + }, + "session_id": "session-scope-7", + "export_attempt_id": "export-attempt-9", + "source_generations": [ + { + "ordinal": 0, + "source_connection_id": "source-notion", + "source_generation_id": "generation-notion-109" + } + ], + "writable_entries": [ + { + "projection_id": "projection-roadmap", + "logical_path": "Projects/Roadmap/page.md", + "source_remote_ids": [ + "page-roadmap" + ], + "delivered_content_sha256": "sha256:5555555555555555555555555555555555555555555555555555555555555555", + "provider_precondition": "opaque-v4", + "effective_actions": [ + "read", + "update" + ], + "baseline_required": true + } + ] + }, + "completion_receipt": { + "versions": { + "session": 2, + "replica": 2, + "export_metadata": 2, + "writable_session_store": 1, + "canonical": 1, + "path": 1, + "changeset": 1 + }, + "session_id": "session-scope-7", + "export_attempt_id": "export-attempt-9", + "source_generations": [ + { + "ordinal": 0, + "source_connection_id": "source-notion", + "source_generation_id": "generation-notion-109" + } + ], + "inventory_sha256": "sha256:025cdbae136931542f7fa881da423e8e1f29a6132cf26ae5f4eea53c53a8ef51", + "writable_metadata_sha256": "sha256:a14c564c8d8950755b30115c4e8c684017662c70e7e03ead0d65a5da132474d2", + "delivered_control_entry_count": 1, + "delivered_file_count": 2, + "delivered_directory_count": 2, + "delivered_archive_entry_count": 5, + "delivered_content_bytes": 17, + "delivered_body_sha256": "sha256:d3dc0915a66e57487d402df0642649990fc61e6128ca8a275dea1e7da93e1381", + "completed_at": "2026-07-23T19:00:04Z" + } +} diff --git a/crates/locality-protocol/fixtures/sealed-export-offer.json b/crates/locality-protocol/fixtures/sealed-export-offer.json index 0100fa18..bf1efe52 100644 --- a/crates/locality-protocol/fixtures/sealed-export-offer.json +++ b/crates/locality-protocol/fixtures/sealed-export-offer.json @@ -30,6 +30,7 @@ "archive_entry_count": 5, "selected_content_bytes": 17, "inventory_sha256": "sha256:025cdbae136931542f7fa881da423e8e1f29a6132cf26ae5f4eea53c53a8ef51", + "writable_metadata_sha256": "sha256:a14c564c8d8950755b30115c4e8c684017662c70e7e03ead0d65a5da132474d2", "sealed_at": "2026-07-23T19:00:01Z", "expires_at": "2026-07-23T19:10:01Z" } diff --git a/crates/locality-protocol/src/lib.rs b/crates/locality-protocol/src/lib.rs index 155a0f31..7921edcf 100644 --- a/crates/locality-protocol/src/lib.rs +++ b/crates/locality-protocol/src/lib.rs @@ -4,7 +4,7 @@ //! This crate owns envelopes, not transport or persistence. In particular it //! contains no HTTP client, database repository, cloud SDK, or host path. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::{Debug, Display, Formatter}; use locality_core::journal::PushOperationId; @@ -35,6 +35,109 @@ pub const EXPORT_V2_FILE_PAX_KEYS: [&str; 6] = [ PAX_EFFECTIVE_ACTIONS, PAX_CONTENT_SHA256, ]; +pub const MAX_EXPORT_V2_PAX_VALUE_BYTES: usize = 4 * 1024; +pub const MAX_EXPORT_V2_FILE_PAX_BYTES: usize = 8 * 1024; +pub const MAX_EXPORT_TERMINAL_CONTROL_BYTES: usize = 4 * 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExportV2FilePaxMetadata { + pub source_connection_id: SourceConnectionId, + pub projection_id: ProjectionId, + pub winning_scope_ordinal: u32, + pub file_kind: ProjectionFileKind, + pub effective_actions: BTreeSet, + pub content_sha256: String, +} + +impl ExportV2FilePaxMetadata { + pub fn from_records(records: &[(String, String)]) -> Option { + let mut locality_records = BTreeMap::new(); + let mut aggregate_bytes = 0_usize; + for (key, value) in records { + if !key.starts_with("locality.") { + continue; + } + if !EXPORT_V2_FILE_PAX_KEYS.contains(&key.as_str()) + || value.len() > MAX_EXPORT_V2_PAX_VALUE_BYTES + { + return None; + } + aggregate_bytes = aggregate_bytes + .checked_add(key.len())? + .checked_add(value.len())?; + if aggregate_bytes > MAX_EXPORT_V2_FILE_PAX_BYTES + || locality_records + .insert(key.as_str(), value.as_str()) + .is_some() + { + return None; + } + } + if locality_records.len() != EXPORT_V2_FILE_PAX_KEYS.len() { + return None; + } + let ordinal = locality_records + .get(PAX_WINNING_SCOPE_ORDINAL)? + .parse::() + .ok()?; + if ordinal.to_string() != *locality_records.get(PAX_WINNING_SCOPE_ORDINAL)? { + return None; + } + let file_kind = projection_file_kind_from_wire_label(locality_records.get(PAX_FILE_KIND)?)?; + if file_kind == ProjectionFileKind::Directory { + return None; + } + let metadata = Self { + source_connection_id: SourceConnectionId::new( + *locality_records.get(PAX_SOURCE_CONNECTION_ID)?, + ), + projection_id: ProjectionId::new(*locality_records.get(PAX_PROJECTION_ID)?), + winning_scope_ordinal: ordinal, + file_kind, + effective_actions: source_actions_from_canonical_pax_value( + locality_records.get(PAX_EFFECTIVE_ACTIONS)?, + )?, + content_sha256: (*locality_records.get(PAX_CONTENT_SHA256)?).to_string(), + }; + metadata.to_records().map(|_| metadata) + } + + pub fn to_records(&self) -> Option> { + if self.source_connection_id.as_str().is_empty() + || self.projection_id.as_str().is_empty() + || self.file_kind == ProjectionFileKind::Directory + || validate_sha256("content_sha256", &self.content_sha256).is_err() + { + return None; + } + let records = vec![ + ( + PAX_SOURCE_CONNECTION_ID, + self.source_connection_id.as_str().to_string(), + ), + (PAX_PROJECTION_ID, self.projection_id.as_str().to_string()), + ( + PAX_WINNING_SCOPE_ORDINAL, + self.winning_scope_ordinal.to_string(), + ), + ( + PAX_FILE_KIND, + projection_file_kind_wire_label(&self.file_kind).to_string(), + ), + ( + PAX_EFFECTIVE_ACTIONS, + canonical_effective_actions_pax_value(&self.effective_actions)?, + ), + (PAX_CONTENT_SHA256, self.content_sha256.clone()), + ]; + let aggregate_bytes = records.iter().try_fold(0_usize, |total, (key, value)| { + (value.len() <= MAX_EXPORT_V2_PAX_VALUE_BYTES) + .then(|| total.checked_add(key.len())?.checked_add(value.len())) + .flatten() + })?; + (aggregate_bytes <= MAX_EXPORT_V2_FILE_PAX_BYTES).then_some(records) + } +} pub const COMPONENT_VERSIONS: ComponentVersions = ComponentVersions { session: 1, @@ -255,6 +358,7 @@ pub struct ReadyReplicaRevision { /// Export implementations receive this fixed value rather than raw selectors, /// user SQL, or client-provided authorization predicates. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct AuthorizedSessionQuery { pub versions: ComponentVersions, pub tenant_id: TenantId, @@ -439,6 +543,7 @@ impl AuthorizedSourceScope { /// An authorized current-head query. Unlike [`AuthorizedSessionQuery`], this /// value names stable source scopes and deliberately does not pin revisions. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ScopeAuthorizedSessionQuery { pub versions: ComponentVersions, pub tenant_id: TenantId, @@ -656,6 +761,7 @@ impl CanonicalExportRecord { content_sha256, .. } => { + validate_nonempty("projection_id", order_key.projection_id.as_str())?; let expected_parent = logical_parent(&order_key.logical_path)?; if order_key.parent_path != expected_parent { return Err(ScopeContractError::InvalidParentPath); @@ -676,6 +782,7 @@ pub fn validate_canonical_export_records( ) -> Result<(), ScopeContractError> { let mut control_count = 0_u64; let mut logical_paths = BTreeSet::new(); + let mut projection_ids = BTreeSet::new(); let mut directory_paths = BTreeSet::new(); let mut previous_key = None; for record in records { @@ -707,6 +814,9 @@ pub fn validate_canonical_export_records( directory_paths.insert(order_key.logical_path.clone()); } CanonicalExportRecord::File { order_key, .. } => { + if !projection_ids.insert(order_key.projection_id.clone()) { + return Err(ScopeContractError::DuplicateValue("projection_id")); + } if order_key.parent_path.as_ref().is_some_and(|parent| { !parent.as_str().eq_ignore_ascii_case(".loc") && !directory_paths.contains(parent) @@ -804,6 +914,7 @@ pub struct DeliveredBodyDigestV2 { hasher: Sha256, expected_file_count: u64, delivered_file_count: u64, + remaining_file_bytes: Option, } impl DeliveredBodyDigestV2 { @@ -815,17 +926,44 @@ impl DeliveredBodyDigestV2 { hasher, expected_file_count, delivered_file_count: 0, + remaining_file_bytes: None, } } - pub fn update_file( + pub fn begin_file( &mut self, projection_id: &ProjectionId, - body: &[u8], + byte_length: u64, ) -> Result<(), ScopeContractError> { + if self.remaining_file_bytes.is_some() { + return Err(ScopeContractError::InventoryDoesNotMatchOffer); + } validate_nonempty("projection_id", projection_id.as_str())?; update_digest_scalar(&mut self.hasher, projection_id.as_str().as_bytes())?; - update_digest_scalar(&mut self.hasher, body)?; + self.hasher.update(byte_length.to_be_bytes()); + self.remaining_file_bytes = Some(byte_length); + Ok(()) + } + + pub fn update_file_chunk(&mut self, chunk: &[u8]) -> Result<(), ScopeContractError> { + let remaining = self + .remaining_file_bytes + .ok_or(ScopeContractError::InventoryDoesNotMatchOffer)?; + let chunk_length = + u64::try_from(chunk.len()).map_err(|_| ScopeContractError::InventoryTooLarge)?; + let next_remaining = remaining + .checked_sub(chunk_length) + .ok_or(ScopeContractError::InventoryDoesNotMatchOffer)?; + self.hasher.update(chunk); + self.remaining_file_bytes = Some(next_remaining); + Ok(()) + } + + pub fn end_file(&mut self) -> Result<(), ScopeContractError> { + if self.remaining_file_bytes != Some(0) { + return Err(ScopeContractError::InventoryDoesNotMatchOffer); + } + self.remaining_file_bytes = None; self.delivered_file_count = self .delivered_file_count .checked_add(1) @@ -833,8 +971,22 @@ impl DeliveredBodyDigestV2 { Ok(()) } + pub fn update_file( + &mut self, + projection_id: &ProjectionId, + body: &[u8], + ) -> Result<(), ScopeContractError> { + let byte_length = + u64::try_from(body.len()).map_err(|_| ScopeContractError::InventoryTooLarge)?; + self.begin_file(projection_id, byte_length)?; + self.update_file_chunk(body)?; + self.end_file() + } + pub fn finish(self) -> Result { - if self.delivered_file_count != self.expected_file_count { + if self.remaining_file_bytes.is_some() + || self.delivered_file_count != self.expected_file_count + { return Err(ScopeContractError::DeliveredBodyCountMismatch { expected: self.expected_file_count, actual: self.delivered_file_count, @@ -1048,6 +1200,7 @@ pub struct SealedExportOffer { pub archive_entry_count: u64, pub selected_content_bytes: u64, pub inventory_sha256: String, + pub writable_metadata_sha256: String, pub sealed_at: String, pub expires_at: String, } @@ -1074,6 +1227,7 @@ impl SealedExportOffer { self.selected_content_bytes, )?; validate_sha256("inventory_sha256", &self.inventory_sha256)?; + validate_sha256("writable_metadata_sha256", &self.writable_metadata_sha256)?; Ok(()) } @@ -1125,6 +1279,7 @@ pub struct ExportCompletionReceipt { pub export_attempt_id: ExportAttemptId, pub source_generations: Vec, pub inventory_sha256: String, + pub writable_metadata_sha256: String, pub delivered_control_entry_count: u64, pub delivered_file_count: u64, pub delivered_directory_count: u64, @@ -1134,6 +1289,185 @@ pub struct ExportCompletionReceipt { pub completed_at: String, } +/// Writable-state metadata carried by the final export-v2 control member. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ScopeAuthorizedWritableExportMetadata { + pub versions: ComponentVersions, + pub session_id: SessionId, + pub export_attempt_id: ExportAttemptId, + pub source_generations: Vec, + pub writable_entries: Vec, +} + +impl ScopeAuthorizedWritableExportMetadata { + pub fn validate_against(&self, offer: &SealedExportOffer) -> Result<(), ScopeContractError> { + validate_writable_metadata_shape(self)?; + if self.session_id != offer.session_id + || self.export_attempt_id != offer.export_attempt_id + || self.source_generations != offer.source_generations + || canonical_writable_metadata_sha256(self)? != offer.writable_metadata_sha256 + { + return Err(ScopeContractError::ReceiptDoesNotMatchOffer); + } + Ok(()) + } +} + +fn validate_writable_metadata_shape( + metadata: &ScopeAuthorizedWritableExportMetadata, +) -> Result<(), ScopeContractError> { + validate_export_versions(&metadata.versions)?; + validate_nonempty("session_id", metadata.session_id.as_str())?; + validate_source_generations(&metadata.source_generations)?; + let mut previous_projection_id = None; + for entry in &metadata.writable_entries { + validate_nonempty("projection_id", entry.projection_id.as_str())?; + if previous_projection_id + .as_ref() + .is_some_and(|previous| previous >= &entry.projection_id) + { + return Err(ScopeContractError::NonCanonicalRecordOrder); + } + previous_projection_id = Some(entry.projection_id.clone()); + if entry.source_remote_ids.is_empty() { + return Err(ScopeContractError::EmptyCollection("source_remote_ids")); + } + if entry + .source_remote_ids + .windows(2) + .any(|pair| pair[0] >= pair[1]) + { + return Err(ScopeContractError::DuplicateValue("source_remote_ids")); + } + if entry.effective_actions.is_empty() { + return Err(ScopeContractError::EmptyCollection("effective_actions")); + } + if !entry + .effective_actions + .iter() + .any(source_action_allows_mutation) + { + return Err(ScopeContractError::InvalidControlRecord); + } + validate_nonempty("provider_precondition", &entry.provider_precondition)?; + validate_sha256("delivered_content_sha256", &entry.delivered_content_sha256)?; + } + Ok(()) +} + +pub fn canonical_writable_metadata_sha256( + metadata: &ScopeAuthorizedWritableExportMetadata, +) -> Result { + validate_writable_metadata_shape(metadata)?; + let mut preimage = b"locality.export.writable-metadata.v2\0".to_vec(); + append_u64(&mut preimage, u64::from(metadata.versions.session))?; + append_u64(&mut preimage, u64::from(metadata.versions.replica))?; + append_u64(&mut preimage, u64::from(metadata.versions.export_metadata))?; + append_u64( + &mut preimage, + u64::from(metadata.versions.writable_session_store), + )?; + append_u64(&mut preimage, u64::from(metadata.versions.canonical))?; + append_u64(&mut preimage, u64::from(metadata.versions.path))?; + append_u64(&mut preimage, u64::from(metadata.versions.changeset))?; + append_text(&mut preimage, metadata.session_id.as_str())?; + append_text(&mut preimage, metadata.export_attempt_id.as_str())?; + append_count(&mut preimage, metadata.source_generations.len())?; + for generation in &metadata.source_generations { + append_u64(&mut preimage, u64::from(generation.ordinal))?; + append_text(&mut preimage, generation.source_connection_id.as_str())?; + append_text(&mut preimage, generation.source_generation_id.as_str())?; + } + append_count(&mut preimage, metadata.writable_entries.len())?; + for entry in &metadata.writable_entries { + append_text(&mut preimage, entry.projection_id.as_str())?; + append_text(&mut preimage, entry.logical_path.as_str())?; + append_count(&mut preimage, entry.source_remote_ids.len())?; + for remote_id in &entry.source_remote_ids { + append_text(&mut preimage, remote_id.as_str())?; + } + append_text(&mut preimage, &entry.delivered_content_sha256)?; + append_text(&mut preimage, &entry.provider_precondition)?; + append_count(&mut preimage, entry.effective_actions.len())?; + for action in &entry.effective_actions { + append_text(&mut preimage, source_action_wire_label(action))?; + } + append_text( + &mut preimage, + if entry.baseline_required { + "true" + } else { + "false" + }, + )?; + } + Ok(format!("sha256:{:x}", Sha256::digest(preimage))) +} + +impl ExportTerminalControlV2 { + pub fn validate_against_inventory( + &self, + offer: &SealedExportOffer, + records: &[CanonicalExportRecord], + ) -> Result<(), ScopeContractError> { + self.validate_against(offer)?; + offer.validate_inventory(records)?; + let files = records + .iter() + .filter_map(|record| match record { + CanonicalExportRecord::File { + order_key, + effective_actions, + content_sha256, + .. + } => Some(( + &order_key.projection_id, + (&order_key.logical_path, effective_actions, content_sha256), + )), + CanonicalExportRecord::Directory { .. } | CanonicalExportRecord::Control { .. } => { + None + } + }) + .collect::>(); + for entry in &self.writable_metadata.writable_entries { + let Some((logical_path, effective_actions, content_sha256)) = + files.get(&entry.projection_id) + else { + return Err(ScopeContractError::InvalidControlRecord); + }; + if *logical_path != &entry.logical_path + || *effective_actions != &entry.effective_actions + || *content_sha256 != &entry.delivered_content_sha256 + { + return Err(ScopeContractError::InvalidControlRecord); + } + } + let mut projection_ids = BTreeSet::new(); + for entry in &self.writable_metadata.writable_entries { + if !projection_ids.insert(entry.projection_id.clone()) { + return Err(ScopeContractError::DuplicateValue("projection_id")); + } + } + Ok(()) + } +} + +/// Exact canonical JSON payload of the unique final `.loc/session.json` +/// export-v2 control member. The member is consumed as transport metadata and +/// is not materialized into the read-only tree. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExportTerminalControlV2 { + pub writable_metadata: ScopeAuthorizedWritableExportMetadata, + pub completion_receipt: ExportCompletionReceipt, +} + +impl ExportTerminalControlV2 { + pub fn validate_against(&self, offer: &SealedExportOffer) -> Result<(), ScopeContractError> { + self.writable_metadata.validate_against(offer)?; + self.completion_receipt.validate_against(offer) + } +} + impl ExportCompletionReceipt { pub fn validate(&self) -> Result<(), ScopeContractError> { validate_export_versions(&self.versions)?; @@ -1147,6 +1481,7 @@ impl ExportCompletionReceipt { self.delivered_archive_entry_count, )?; validate_sha256("inventory_sha256", &self.inventory_sha256)?; + validate_sha256("writable_metadata_sha256", &self.writable_metadata_sha256)?; validate_sha256("delivered_body_sha256", &self.delivered_body_sha256)?; Ok(()) } @@ -1158,6 +1493,7 @@ impl ExportCompletionReceipt { || self.export_attempt_id != offer.export_attempt_id || self.source_generations != offer.source_generations || self.inventory_sha256 != offer.inventory_sha256 + || self.writable_metadata_sha256 != offer.writable_metadata_sha256 || self.delivered_control_entry_count != offer.control_entry_count || self.delivered_file_count != offer.file_count || self.delivered_directory_count != offer.directory_count @@ -1705,6 +2041,13 @@ pub fn source_action_from_wire_label(value: &str) -> Option { } } +pub fn source_action_allows_mutation(action: &SourceAction) -> bool { + !matches!( + action, + SourceAction::Read | SourceAction::Search | SourceAction::DownloadAttachment + ) +} + /// Encodes the effective actions carried by an export-v2 file PAX record. /// /// The wire value is a compact JSON array whose labels are sorted by their @@ -1848,6 +2191,8 @@ pub const SEALED_EXPORT_OFFER_GOLDEN_JSON: &[u8] = include_bytes!("../fixtures/sealed-export-offer.json"); pub const EXPORT_COMPLETION_RECEIPT_GOLDEN_JSON: &[u8] = include_bytes!("../fixtures/export-completion-receipt.json"); +pub const EXPORT_TERMINAL_CONTROL_V2_GOLDEN_JSON: &[u8] = + include_bytes!("../fixtures/export-terminal-control-v2.json"); pub const CANONICAL_EXPORT_RECORDS_GOLDEN_JSON: &[u8] = include_bytes!("../fixtures/canonical-export-records.json"); pub const CANONICAL_EXPORT_INVENTORY_GOLDEN_JSON: &[u8] = diff --git a/crates/locality-protocol/tests/golden.rs b/crates/locality-protocol/tests/golden.rs index ad3b9829..69ef143c 100644 --- a/crates/locality-protocol/tests/golden.rs +++ b/crates/locality-protocol/tests/golden.rs @@ -21,20 +21,21 @@ use locality_protocol::{ ChangesetSourceObject, ClientValidationResult, CompatibleAuthorizedSessionQuery, ComponentVersions, ContentVersionContract, DELIVERED_COUNT_GOLDEN_JSON, DeliveredBodyDigestV2, DeliveredChangesetBase, DeliveredCount, EXPORT_ATTEMPT_REQUEST_GOLDEN_JSON, - EXPORT_COMPLETION_RECEIPT_GOLDEN_JSON, EditedCanonicalBody, ExportAttemptLimits, - ExportAttemptRequest, ExportCompletionReceipt, FRESHNESS_STATUS_GOLDEN_JSON, - FreshnessRequirement, NotionScopeKind, ORDERED_EXPORT_ROWS_GOLDEN_JSON, - OpaqueBootstrapExchangeRequest, OpaqueSessionStatusRequest, OrderedExportRow, - OrderedSourceGeneration, PROJECTION_VERSION_GOLDEN_JSON, ProjectionVersionContract, - ProviderSourceScopeSelector, READY_REPLICA_REVISION_GOLDEN_JSON, ReadyReplicaRevision, - ReplicaFreshnessState, ReplicaFreshnessStatus, SANDBOX_SESSION_STATUS_GOLDEN_JSON, - SCOPE_AUTHORIZED_COMPONENT_VERSIONS, SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, - SEALED_EXPORT_OFFER_GOLDEN_JSON, SESSION_PROTOCOL_ERROR_GOLDEN_JSON, - SOURCE_VERSION_GOLDEN_JSON, SandboxSessionState, SandboxSessionStatus, - ScopeAuthorizedSessionQuery, ScopeContractError, SealedExportOffer, SessionCapability, - SessionErrorCode, SessionProtocolError, SessionReplicaRevision, SourceVersionContract, - StaleSessionBehavior, TAR_EXPORT_METADATA_GOLDEN_JSON, TAR_EXPORT_OFFER_GOLDEN_JSON, - TarContentEncoding, TarExportMetadata, TarExportOffer, WRITABLE_EXPORT_METADATA_GOLDEN_JSON, + EXPORT_COMPLETION_RECEIPT_GOLDEN_JSON, EXPORT_TERMINAL_CONTROL_V2_GOLDEN_JSON, + EditedCanonicalBody, ExportAttemptLimits, ExportAttemptRequest, ExportCompletionReceipt, + ExportTerminalControlV2, FRESHNESS_STATUS_GOLDEN_JSON, FreshnessRequirement, NotionScopeKind, + ORDERED_EXPORT_ROWS_GOLDEN_JSON, OpaqueBootstrapExchangeRequest, OpaqueSessionStatusRequest, + OrderedExportRow, OrderedSourceGeneration, PROJECTION_VERSION_GOLDEN_JSON, + ProjectionVersionContract, ProviderSourceScopeSelector, READY_REPLICA_REVISION_GOLDEN_JSON, + ReadyReplicaRevision, ReplicaFreshnessState, ReplicaFreshnessStatus, + SANDBOX_SESSION_STATUS_GOLDEN_JSON, SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, SEALED_EXPORT_OFFER_GOLDEN_JSON, + SESSION_PROTOCOL_ERROR_GOLDEN_JSON, SOURCE_VERSION_GOLDEN_JSON, SandboxSessionState, + SandboxSessionStatus, ScopeAuthorizedSessionQuery, ScopeAuthorizedWritableExportMetadata, + ScopeContractError, SealedExportOffer, SessionCapability, SessionErrorCode, + SessionProtocolError, SessionReplicaRevision, SourceVersionContract, StaleSessionBehavior, + TAR_EXPORT_METADATA_GOLDEN_JSON, TAR_EXPORT_OFFER_GOLDEN_JSON, TarContentEncoding, + TarExportMetadata, TarExportOffer, WRITABLE_EXPORT_METADATA_GOLDEN_JSON, WritableExportMetadata, WritableMetadataEntry, validate_canonical_export_records, }; use serde::de::DeserializeOwned; @@ -91,6 +92,12 @@ fn scope_authorized_export_contracts_are_exact_golden_bytes() { .expect("completion matches sealed metadata selection"); assert_exact_round_trip(EXPORT_COMPLETION_RECEIPT_GOLDEN_JSON, &receipt); + let control = export_terminal_control_v2(); + control + .validate_against_inventory(&offer, &canonical_export_records()) + .expect("terminal control matches sealed inventory"); + assert_exact_round_trip(EXPORT_TERMINAL_CONTROL_V2_GOLDEN_JSON, &control); + let records = canonical_export_records(); validate_canonical_export_records(&records).expect("canonical record order"); let preimage = locality_protocol::canonical_export_inventory_preimage(&records) @@ -379,6 +386,88 @@ fn export_v2_pax_wire_contract_is_exact_and_round_trips() { "accepted non-canonical effective actions: {invalid}" ); } + + let metadata = locality_protocol::ExportV2FilePaxMetadata { + source_connection_id: SourceConnectionId::new("source-notion"), + projection_id: ProjectionId::new("projection-roadmap"), + winning_scope_ordinal: 3, + file_kind: ProjectionFileKind::Markdown, + effective_actions: BTreeSet::from([SourceAction::Read, SourceAction::Update]), + content_sha256: format!("sha256:{}", "5".repeat(64)), + }; + let records = metadata + .to_records() + .expect("canonical PAX records") + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect::>(); + assert_eq!( + locality_protocol::ExportV2FilePaxMetadata::from_records(&records), + Some(metadata) + ); + let mut duplicate = records.clone(); + duplicate.push(records[0].clone()); + assert!(locality_protocol::ExportV2FilePaxMetadata::from_records(&duplicate).is_none()); + let mut unknown = records.clone(); + unknown.push(("locality.future".to_string(), "value".to_string())); + assert!(locality_protocol::ExportV2FilePaxMetadata::from_records(&unknown).is_none()); + let mut noncanonical_ordinal = records; + noncanonical_ordinal + .iter_mut() + .find(|(key, _)| key == locality_protocol::PAX_WINNING_SCOPE_ORDINAL) + .expect("ordinal") + .1 = "03".to_string(); + assert!( + locality_protocol::ExportV2FilePaxMetadata::from_records(&noncanonical_ordinal).is_none() + ); +} + +#[test] +fn export_v2_order_and_body_digest_are_streaming_exact() { + let root = CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: None, + logical_path: LogicalPath::new("README.md").expect("path"), + projection_id: ProjectionId::new("projection-root"), + }; + let nested = CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: Some(LogicalPath::new("Projects").expect("path")), + logical_path: LogicalPath::new("Projects/page.md").expect("path"), + projection_id: ProjectionId::new("projection-nested"), + }; + assert!(root < nested, "root files must match SQL NULLS FIRST order"); + + let mut whole = DeliveredBodyDigestV2::new(1); + whole + .update_file(&ProjectionId::new("projection-root"), b"abcdefghij") + .expect("whole body"); + let whole = whole.finish().expect("whole digest"); + + let mut streamed = DeliveredBodyDigestV2::new(1); + streamed + .begin_file(&ProjectionId::new("projection-root"), 10) + .expect("begin file"); + for chunk in [b"ab".as_slice(), b"cdef".as_slice(), b"ghij".as_slice()] { + streamed.update_file_chunk(chunk).expect("body chunk"); + } + streamed.end_file().expect("end file"); + assert_eq!(streamed.finish().expect("streamed digest"), whole); +} + +#[test] +fn mixed_legacy_and_scope_session_queries_are_rejected() { + let mut mixed = + serde_json::from_slice::(SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON) + .expect("scope fixture"); + mixed + .as_object_mut() + .expect("object") + .insert("replica_revisions".to_string(), serde_json::json!([])); + assert!( + serde_json::from_value::(mixed).is_err(), + "mixed authority shapes must not be silently reinterpreted" + ); } #[test] @@ -606,6 +695,10 @@ fn source_generations() -> Vec { } fn sealed_export_offer() -> SealedExportOffer { + let writable_metadata_sha256 = locality_protocol::canonical_writable_metadata_sha256( + &scope_authorized_writable_metadata(), + ) + .expect("writable metadata digest"); SealedExportOffer { versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, session_id: SessionId::new("session-scope-7"), @@ -621,6 +714,7 @@ fn sealed_export_offer() -> SealedExportOffer { selected_content_bytes: 17, inventory_sha256: "sha256:025cdbae136931542f7fa881da423e8e1f29a6132cf26ae5f4eea53c53a8ef51" .to_string(), + writable_metadata_sha256, sealed_at: "2026-07-23T19:00:01Z".to_string(), expires_at: "2026-07-23T19:10:01Z".to_string(), } @@ -641,6 +735,10 @@ fn export_completion_receipt() -> ExportCompletionReceipt { source_generations: source_generations(), inventory_sha256: "sha256:025cdbae136931542f7fa881da423e8e1f29a6132cf26ae5f4eea53c53a8ef51" .to_string(), + writable_metadata_sha256: locality_protocol::canonical_writable_metadata_sha256( + &scope_authorized_writable_metadata(), + ) + .expect("writable metadata digest"), delivered_control_entry_count: 1, delivered_file_count: 2, delivered_directory_count: 2, @@ -651,6 +749,31 @@ fn export_completion_receipt() -> ExportCompletionReceipt { } } +fn scope_authorized_writable_metadata() -> ScopeAuthorizedWritableExportMetadata { + ScopeAuthorizedWritableExportMetadata { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: SessionId::new("session-scope-7"), + export_attempt_id: ExportAttemptId::new("export-attempt-9").expect("attempt id"), + source_generations: source_generations(), + writable_entries: vec![WritableMetadataEntry { + projection_id: ProjectionId::new("projection-roadmap"), + logical_path: LogicalPath::new("Projects/Roadmap/page.md").expect("path"), + source_remote_ids: vec![RemoteId::new("page-roadmap")], + delivered_content_sha256: format!("sha256:{}", "5".repeat(64)), + provider_precondition: "opaque-v4".to_string(), + effective_actions: BTreeSet::from([SourceAction::Read, SourceAction::Update]), + baseline_required: true, + }], + } +} + +fn export_terminal_control_v2() -> ExportTerminalControlV2 { + ExportTerminalControlV2 { + writable_metadata: scope_authorized_writable_metadata(), + completion_receipt: export_completion_receipt(), + } +} + fn canonical_export_records() -> Vec { vec![ CanonicalExportRecord::Directory { diff --git a/docs/cloud-sandbox-data-plane.md b/docs/cloud-sandbox-data-plane.md index 3251b016..4dcdae09 100644 --- a/docs/cloud-sandbox-data-plane.md +++ b/docs/cloud-sandbox-data-plane.md @@ -567,8 +567,7 @@ freshness observation, and publication state. Projection content uses immutable version rows plus current/version-range mappings, so publishing a generation does not duplicate a physical corpus. Its immutable manifest retains the exact scope memberships, logical paths, connector/source-supported action ceilings, -content references, - hashes, sizes, connector/source-supported action ceilings, and exact attachment +content references, hashes, sizes, and exact attachment object versions required for retries and audit after a newer head publishes. ```text @@ -1189,7 +1188,7 @@ LEFT JOIN content_versions AS c AND c.source_connection_id = e.source_connection_id AND c.content_storage_id = e.content_storage_id ORDER BY e.winning_scope_ordinal, - e.parent_path COLLATE "C", + e.parent_path COLLATE "C" NULLS FIRST, e.logical_path COLLATE "C", e.projection_id; ``` @@ -1320,19 +1319,22 @@ Content-Encoding: zstd order, so every unique parent precedes descendants; - each selected projection is record class 1 and becomes one ordinary tar member with its logical path, size, and non-executable mode. File members follow - `(winning_scope_ordinal, parent_path COLLATE "C", logical_path COLLATE "C", - projection_id)` order; + `(winning_scope_ordinal, parent_path COLLATE "C" NULLS FIRST, + logical_path COLLATE "C", projection_id)` order; - each v2 file member carries bounded transient POSIX PAX fields for source connection ID, projection ID, winning scope ordinal, file kind, sorted effective actions, and content SHA-256. Public constants and parsers define - their exact names/encoding. They allow the client to recompute the inventory - and body digests but are never materialized or inserted into SQLite; directory - and control members carry no Locality PAX fields; + their exact names/encoding and bounded per-value/per-file size. Duplicate or + unknown `locality.*` fields are rejected. They allow the client to recompute + the inventory and body digests but are never materialized or inserted into + SQLite; directory and control members carry no Locality PAX fields; - v1 retains its reserved first writable-metadata member unchanged. V2 instead requires exactly one final record-class-2 `.loc/session.json` member combining that writable metadata with the completion receipt. It is bounded, canonical JSON, regular mode `0444`, consumed into the small SQLite store, never exposed - in the mount, and has no emitted `.loc/` directory header; + in the mount, and has no emitted `.loc/` directory header. The offer and + receipt both seal a domain-separated digest of the writable metadata so + source identities and write preconditions cannot change after preflight; - read-only entries create no local metadata or shadow rows; - selected large attachment references are fetched by the exporter and appended to the same stream under bounded size/scanning policy; and @@ -2946,8 +2948,9 @@ verification, and dedicated-cell operational runbooks are complete. prunes the expected hash partitions, uses current-head versus historical revision paths correctly, and joins bodies by `content_storage_id` only after authorization/filter selection. -- Ordered-ingest/export fixtures prove the single reserved control member is - first and counted, directory headers then use parent-first `(depth, +- Ordered-ingest/export fixtures prove v1 keeps its reserved control member + first, while v2 emits parent-first directory headers, then files, then exactly + one final counted control member. Directory headers use `(depth, logical_path)` order, and file members use `(winning_scope_ordinal, parent_path, logical_path, projection_id)` under bytewise `C` collation. The order does not change logical paths or canonical From 606ea1acfa48828bf2b362d95e701043d11fba78 Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 15:49:25 -0700 Subject: [PATCH 07/14] Canonicalize writable action digests --- crates/locality-protocol/src/lib.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/locality-protocol/src/lib.rs b/crates/locality-protocol/src/lib.rs index 7921edcf..1ecbad99 100644 --- a/crates/locality-protocol/src/lib.rs +++ b/crates/locality-protocol/src/lib.rs @@ -1389,8 +1389,14 @@ pub fn canonical_writable_metadata_sha256( append_text(&mut preimage, &entry.delivered_content_sha256)?; append_text(&mut preimage, &entry.provider_precondition)?; append_count(&mut preimage, entry.effective_actions.len())?; - for action in &entry.effective_actions { - append_text(&mut preimage, source_action_wire_label(action))?; + let mut action_labels = entry + .effective_actions + .iter() + .map(source_action_wire_label) + .collect::>(); + action_labels.sort_unstable(); + for action_label in action_labels { + append_text(&mut preimage, action_label)?; } append_text( &mut preimage, From c79f4c2632596d0b757eb1f93fc9bf29da198b41 Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 15:54:08 -0700 Subject: [PATCH 08/14] feat(sandbox): materialize scope export attempts --- crates/loc-cli/src/sandbox.rs | 278 ++++++++++++- crates/loc-cli/tests/sandbox.rs | 291 ++++++++++++- crates/localityd/src/replica_materializer.rs | 387 ++++++++++++++++++ .../localityd/tests/replica_materializer.rs | 360 ++++++++++++++++ 4 files changed, 1298 insertions(+), 18 deletions(-) diff --git a/crates/loc-cli/src/sandbox.rs b/crates/loc-cli/src/sandbox.rs index 3cc87b5b..317d7799 100644 --- a/crates/loc-cli/src/sandbox.rs +++ b/crates/loc-cli/src/sandbox.rs @@ -16,13 +16,15 @@ use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; use locality_protocol::{ - OpaqueBootstrapExchangeRequest, SandboxSessionState, SandboxSessionStatus, SessionCapability, - SessionErrorCode, TarContentEncoding, TarExportOffer, + ExportAttemptLimits, ExportAttemptRequest, OpaqueBootstrapExchangeRequest, + SCOPE_AUTHORIZED_COMPONENT_VERSIONS, SandboxSessionState, SandboxSessionStatus, + SealedExportOffer, SessionCapability, SessionErrorCode, TarContentEncoding, TarExportOffer, }; use localityd::remote_truth::{ReplicaArchive, ReplicaArchiveEncoding}; use localityd::replica_materializer::{ ExpectedReplicaMaterializationReceipt, ReplicaMaterializationLimits, ReplicaMaterializationSummary, materialize_replica_archive_with_expected_receipt, + materialize_scope_authorized_replica_archive, }; use reqwest::StatusCode; use reqwest::blocking::{Client, Response}; @@ -440,10 +442,36 @@ fn run_sandbox_init_internal( validate_capability(&capability)?; let status = client.session_status(&capability)?; mark_profile(&mut profile, PROFILE_SESSION_STATUS); - let (offer, expected_receipt) = validate_status(&capability, &status)?; - validate_encoding_preference(offer, content_encoding)?; - let limits = limits_for_offer(offer)?; - let (encoding, response) = client.open_export(&capability, offer, content_encoding)?; + let session = validate_status(&capability, &status)?; + let (encoding, response, limits, validation) = match session { + ValidatedSandboxSession::Legacy { + offer, + expected_receipt, + } => { + validate_encoding_preference(offer, content_encoding)?; + let limits = limits_for_offer(offer)?; + let (encoding, response) = client.open_export(&capability, offer, content_encoding)?; + ( + encoding, + response, + limits, + ExportValidation::Legacy(expected_receipt), + ) + } + ValidatedSandboxSession::ScopeAuthorized => { + let request = export_attempt_request(&capability, content_encoding)?; + let offer = client.create_export_attempt(&capability, &request)?; + validate_scope_offer(&capability, &request, &offer)?; + let limits = limits_for_scope_offer(&offer)?; + let (encoding, response) = client.open_export_attempt(&capability, &offer)?; + ( + encoding, + response, + limits, + ExportValidation::ScopeAuthorized(offer), + ) + } + }; mark_profile(&mut profile, PROFILE_EXPORT_OPEN_HEADERS); let (body, mut producer) = spawn_export_read_ahead(response).map_err(|error| SandboxInitError::Http { @@ -452,9 +480,20 @@ fn run_sandbox_init_internal( })?; let profiled_body = ProfiledExportBody::new(body, profile.as_deref_mut()); let archive = ReplicaArchive::new(encoding, profiled_body); - let materialization = - materialize_replica_archive_with_expected_receipt(archive, &root, limits, expected_receipt) - .map_err(|error| SandboxInitError::Materialization(error.to_string())); + let materialization = match validation { + ExportValidation::Legacy(expected_receipt) => { + materialize_replica_archive_with_expected_receipt( + archive, + &root, + limits, + expected_receipt, + ) + } + ExportValidation::ScopeAuthorized(offer) => { + materialize_scope_authorized_replica_archive(archive, &root, limits, &offer) + } + } + .map_err(|error| SandboxInitError::Materialization(error.to_string())); let producer_outcome = producer.join(); mark_profile(&mut profile, PROFILE_STREAM_DECODE_MATERIALIZE); @@ -717,10 +756,23 @@ fn validate_capability(capability: &SessionCapability) -> Result<(), SandboxInit Ok(()) } +enum ValidatedSandboxSession<'a> { + Legacy { + offer: &'a TarExportOffer, + expected_receipt: ExpectedReplicaMaterializationReceipt, + }, + ScopeAuthorized, +} + +enum ExportValidation { + Legacy(ExpectedReplicaMaterializationReceipt), + ScopeAuthorized(SealedExportOffer), +} + fn validate_status<'a>( capability: &SessionCapability, status: &'a SandboxSessionStatus, -) -> Result<(&'a TarExportOffer, ExpectedReplicaMaterializationReceipt), SandboxInitError> { +) -> Result, SandboxInitError> { status .versions .validate_required() @@ -737,6 +789,20 @@ fn validate_status<'a>( if status.error.is_some() { return Err(SandboxInitError::InvalidReadySession("error is present")); } + if status.versions.session >= 2 { + if status.versions.replica < 2 || status.versions.export_metadata < 2 { + return Err(SandboxInitError::ComponentVersion( + "scope-authorized session requires replica and export-metadata version 2" + .to_string(), + )); + } + if status.export_offer.is_some() { + return Err(SandboxInitError::InvalidReadySession( + "scope-authorized status contains a legacy export offer", + )); + } + return Ok(ValidatedSandboxSession::ScopeAuthorized); + } let offer = status .export_offer .as_ref() @@ -744,7 +810,81 @@ fn validate_status<'a>( "export offer is missing", ))?; let expected_receipt = validate_offer(offer)?; - Ok((offer, expected_receipt)) + Ok(ValidatedSandboxSession::Legacy { + offer, + expected_receipt, + }) +} + +fn export_attempt_request( + capability: &SessionCapability, + preference: SandboxContentEncodingPreference, +) -> Result { + let defaults = ReplicaMaterializationLimits::default(); + let request = ExportAttemptRequest { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + opaque_session_capability: capability.opaque_capability.clone(), + idempotency_key: random_export_idempotency_key()?, + content_encoding: match preference { + SandboxContentEncodingPreference::Automatic + | SandboxContentEncodingPreference::Zstd => TarContentEncoding::Zstd, + SandboxContentEncodingPreference::Identity => TarContentEncoding::Identity, + }, + limits: ExportAttemptLimits { + max_files: defaults.max_entries.saturating_sub(1), + max_directories: defaults.max_entries.saturating_sub(1), + max_content_bytes: defaults.max_disk_bytes, + }, + }; + request.validate().map_err(|_| { + SandboxInitError::InvalidExportOffer("client export-attempt request is invalid") + })?; + Ok(request) +} + +fn random_export_idempotency_key() -> Result { + let mut random = [0_u8; 32]; + rustls::crypto::ring::default_provider() + .secure_random + .fill(&mut random) + .map_err(|_| SandboxInitError::Http { + operation: "export-attempt idempotency-key generation", + detail: "secure randomness is unavailable".to_string(), + })?; + Ok(format!("loc-export-v2-{}", lower_hex(&random))) +} + +fn validate_scope_offer( + capability: &SessionCapability, + request: &ExportAttemptRequest, + offer: &SealedExportOffer, +) -> Result<(), SandboxInitError> { + offer + .versions + .validate_required() + .map_err(|error| SandboxInitError::ComponentVersion(error.to_string()))?; + offer + .validate() + .map_err(|_| SandboxInitError::InvalidExportOffer("scope-authorized offer is invalid"))?; + if offer.session_id != capability.session_id { + return Err(SandboxInitError::SessionIdMismatch); + } + if offer.media_type != TAR_MEDIA_TYPE { + return Err(SandboxInitError::InvalidExportOffer( + "media type must be application/x-tar", + )); + } + if offer.content_encoding != request.content_encoding { + return Err(SandboxInitError::InvalidExportOffer( + "content encoding does not match the export-attempt request", + )); + } + if offer.limits != request.limits { + return Err(SandboxInitError::InvalidExportOffer( + "limits do not match the export-attempt request", + )); + } + Ok(()) } fn validate_offer( @@ -844,6 +984,31 @@ fn limits_for_offer( }) } +fn limits_for_scope_offer( + offer: &SealedExportOffer, +) -> Result { + let defaults = ReplicaMaterializationLimits::default(); + if offer.archive_entry_count > defaults.max_entries { + return Err(SandboxInitError::ExportLimit { + limit: "entry count", + offered: offer.archive_entry_count, + maximum: defaults.max_entries, + }); + } + if offer.selected_content_bytes > defaults.max_disk_bytes { + return Err(SandboxInitError::ExportLimit { + limit: "content bytes", + offered: offer.selected_content_bytes, + maximum: defaults.max_disk_bytes, + }); + } + Ok(ReplicaMaterializationLimits { + max_entries: offer.archive_entry_count, + max_disk_bytes: offer.selected_content_bytes, + ..defaults + }) +} + fn validate_encoding_preference( offer: &TarExportOffer, preference: SandboxContentEncodingPreference, @@ -1055,6 +1220,66 @@ impl SandboxHttpClient { Ok((encoding, response)) } + fn create_export_attempt( + &self, + capability: &SessionCapability, + request: &ExportAttemptRequest, + ) -> Result { + let response = self + .client + .post(self.export_attempts_url(capability.session_id.as_str())) + .header(ACCEPT, JSON_MEDIA_TYPE) + .bearer_auth(&capability.opaque_capability) + .json(request) + .send() + .map_err(|error| SandboxInitError::Http { + operation: "export-attempt creation", + detail: error.without_url().to_string(), + })?; + read_json_response(response, "export-attempt creation") + } + + fn open_export_attempt( + &self, + capability: &SessionCapability, + offer: &SealedExportOffer, + ) -> Result<(ReplicaArchiveEncoding, Response), SandboxInitError> { + let response = self + .export_client + .get(self.export_attempt_url( + capability.session_id.as_str(), + offer.export_attempt_id.as_str(), + )) + .header(ACCEPT, TAR_MEDIA_TYPE) + .header( + ACCEPT_ENCODING, + match offer.content_encoding { + TarContentEncoding::Identity => "identity", + TarContentEncoding::Zstd => "zstd", + }, + ) + .bearer_auth(&capability.opaque_capability) + .send() + .map_err(|error| SandboxInitError::Http { + operation: "export-attempt stream", + detail: error.without_url().to_string(), + })?; + ensure_success(&response, "export-attempt stream")?; + require_media_type(response.headers(), "export-attempt stream", TAR_MEDIA_TYPE)?; + let encoding = response_encoding(response.headers())?; + if protocol_encoding(encoding) != offer.content_encoding { + return Err(SandboxInitError::UnsupportedExportEncoding(format!( + "{} (sealed {})", + encoding_name(encoding), + match offer.content_encoding { + TarContentEncoding::Identity => "identity", + TarContentEncoding::Zstd => "zstd", + } + ))); + } + Ok((encoding, response)) + } + fn sessions_url(&self) -> reqwest::Url { endpoint_url(&self.api_url, &["v1", "sessions"]) } @@ -1066,6 +1291,27 @@ impl SandboxHttpClient { fn export_url(&self, session_id: &str) -> reqwest::Url { endpoint_url(&self.api_url, &["v1", "sessions", session_id, "export"]) } + + fn export_attempts_url(&self, session_id: &str) -> reqwest::Url { + endpoint_url( + &self.api_url, + &["v1", "sessions", session_id, "export-attempts"], + ) + } + + fn export_attempt_url(&self, session_id: &str, attempt_id: &str) -> reqwest::Url { + endpoint_url( + &self.api_url, + &[ + "v1", + "sessions", + session_id, + "export-attempts", + attempt_id, + "export", + ], + ) + } } fn derive_idempotency_key(token: &SandboxBootstrapToken) -> String { @@ -1073,13 +1319,17 @@ fn derive_idempotency_key(token: &SandboxBootstrapToken) -> String { hasher.update(BOOTSTRAP_IDEMPOTENCY_DOMAIN); hasher.update(token.expose().as_bytes()); let digest = hasher.finalize(); - let mut encoded = [0_u8; 64]; + lower_hex(&digest) +} + +fn lower_hex(bytes: &[u8]) -> String { + let mut encoded = vec![0_u8; bytes.len() * 2]; const LOWER_HEX: &[u8; 16] = b"0123456789abcdef"; - for (index, byte) in digest.into_iter().enumerate() { + for (index, byte) in bytes.iter().copied().enumerate() { encoded[index * 2] = LOWER_HEX[usize::from(byte >> 4)]; encoded[index * 2 + 1] = LOWER_HEX[usize::from(byte & 0x0f)]; } - String::from_utf8(encoded.to_vec()).expect("lowercase hexadecimal is valid UTF-8") + String::from_utf8(encoded).expect("lowercase hexadecimal is valid UTF-8") } fn has_retry_remaining(attempt: usize) -> bool { diff --git a/crates/loc-cli/tests/sandbox.rs b/crates/loc-cli/tests/sandbox.rs index eeb6bc4e..448db2ab 100644 --- a/crates/loc-cli/tests/sandbox.rs +++ b/crates/loc-cli/tests/sandbox.rs @@ -13,11 +13,21 @@ use loc_cli::sandbox::{ SandboxBootstrapToken, SandboxContentEncodingPreference, SandboxInitError, SandboxInitOptions, resolve_bootstrap_token, run_sandbox_init, run_sandbox_init_with_encoding, }; -use locality_core::portable::SessionId; +use locality_core::portable::{ + ExportAttemptId, LogicalPath, ProjectionFileKind, ProjectionId, SessionId, SourceAction, + SourceConnectionId, SourceGenerationId, +}; use locality_protocol::{ - COMPONENT_VERSIONS, ComponentVersions, FreshnessRequirement, OpaqueBootstrapExchangeRequest, - SandboxSessionState, SandboxSessionStatus, SessionCapability, SessionErrorCode, - SessionProtocolError, StaleSessionBehavior, TarContentEncoding, TarExportOffer, + COMPONENT_VERSIONS, CanonicalControlOrderKey, CanonicalExportRecord, CanonicalFileOrderKey, + ComponentVersions, DeliveredBodyDigestV2, ExportAttemptLimits, ExportAttemptRequest, + ExportCompletionReceipt, ExportTerminalControlV2, FreshnessRequirement, + OpaqueBootstrapExchangeRequest, OrderedSourceGeneration, PAX_CONTENT_SHA256, + PAX_EFFECTIVE_ACTIONS, PAX_FILE_KIND, PAX_PROJECTION_ID, PAX_SOURCE_CONNECTION_ID, + PAX_WINNING_SCOPE_ORDINAL, SCOPE_AUTHORIZED_COMPONENT_VERSIONS, SandboxSessionState, + SandboxSessionStatus, ScopeAuthorizedWritableExportMetadata, SealedExportOffer, + SessionCapability, SessionErrorCode, SessionProtocolError, StaleSessionBehavior, + TarContentEncoding, TarExportOffer, canonical_export_inventory_sha256, + canonical_writable_metadata_sha256, }; use localityd::replica_materializer::ReplicaMaterializationLimits; use sha2::{Digest, Sha256}; @@ -319,6 +329,137 @@ fn zstd_bootstrap_streams_into_the_shared_materializer() { ); } +#[test] +fn scope_authorized_export_attempts_materialize_identity_and_zstd() { + let mut idempotency_keys = Vec::new(); + for (label, preference, protocol_encoding, header_encoding) in [ + ( + "scope-identity", + SandboxContentEncodingPreference::Identity, + TarContentEncoding::Identity, + "identity", + ), + ( + "scope-zstd", + SandboxContentEncodingPreference::Zstd, + TarContentEncoding::Zstd, + "zstd", + ), + ] { + let directory = TestDirectory::new(label); + let capability = capability(); + let status = scope_ready_status(capability.session_id.clone()); + let (offer, tar) = scope_export_fixture(protocol_encoding); + let response_body = match protocol_encoding { + TarContentEncoding::Identity => tar, + TarContentEncoding::Zstd => { + zstd::stream::encode_all(tar.as_slice(), 1).expect("compress scope tar") + } + }; + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ResponseFixture::json(&offer), + ResponseFixture::export(header_encoding, response_body), + ]); + + let report = run_sandbox_init_with_encoding( + SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }, + SandboxBootstrapToken::new("scope-bootstrap-secret").expect("token"), + preference, + ) + .expect("materialize scope-authorized export"); + + assert_eq!(report.content_encoding, header_encoding); + assert_eq!(report.entries, 2); + assert_eq!(report.files, 1); + assert_eq!(report.directories, 0); + assert_eq!(report.materialized_bytes, 9); + assert_eq!( + fs::read(directory.root().join("readme.md")).expect("read scope file"), + b"scope v2\n" + ); + assert!(!directory.root().join(".loc").exists()); + + let _bootstrap = server.request(); + let _status = server.request(); + let attempt = server.request(); + assert_eq!(attempt.method, "POST"); + assert_eq!(attempt.path, "/v1/sessions/session-7/export-attempts"); + assert_eq!( + attempt.headers.get("authorization").unwrap(), + "Bearer capability-secret" + ); + let request: ExportAttemptRequest = + serde_json::from_slice(&attempt.body).expect("decode export-attempt request"); + request.validate().expect("valid export-attempt request"); + assert_eq!(request.content_encoding, protocol_encoding); + assert_eq!(request.opaque_session_capability, "capability-secret"); + assert!(request.idempotency_key.starts_with("loc-export-v2-")); + assert_eq!(request.idempotency_key.len(), 78); + idempotency_keys.push(request.idempotency_key); + + let export = server.request(); + assert_eq!(export.method, "GET"); + assert_eq!( + export.path, + "/v1/sessions/session-7/export-attempts/attempt-scope/export" + ); + assert_eq!( + export.headers.get("accept-encoding").unwrap(), + header_encoding + ); + } + assert_ne!(idempotency_keys[0], idempotency_keys[1]); +} + +#[test] +fn malformed_scope_offer_fails_before_stream_without_identifier_leaks() { + let directory = TestDirectory::new("scope-malformed-offer"); + let capability = capability(); + let status = scope_ready_status(capability.session_id.clone()); + let (mut offer, _tar) = scope_export_fixture(TarContentEncoding::Zstd); + offer.archive_entry_count = 1; + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ResponseFixture::json(&offer), + ]); + + let error = run_sandbox_init( + SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }, + SandboxBootstrapToken::new("scope-bootstrap-secret").expect("token"), + ) + .expect_err("malformed scope offer must fail before export"); + + assert_eq!(error.code(), "backend_protocol_invalid"); + let display = error.to_string(); + for secret_or_id in [ + "scope-bootstrap-secret", + "capability-secret", + "session-7", + "attempt-scope", + "source-notion", + "generation-scope", + ] { + assert!( + !display.contains(secret_or_id), + "error leaked {secret_or_id}" + ); + } + assert!(!directory.root().exists()); + let _ = server.request(); + let _ = server.request(); + let _ = server.request(); + server.assert_no_request(); +} + #[test] fn export_bytes_are_staged_while_the_http_response_is_still_streaming() { let directory = TestDirectory::new("streaming-overlap"); @@ -1267,6 +1408,148 @@ fn ready_status( } } +fn scope_ready_status(session_id: SessionId) -> SandboxSessionStatus { + SandboxSessionStatus { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id, + state: SandboxSessionState::Ready, + freshness_requirement: freshness_requirement(), + replicas: Vec::new(), + export_offer: None, + error: None, + updated_at: "2026-07-23T20:00:00Z".to_string(), + } +} + +fn scope_export_fixture(encoding: TarContentEncoding) -> (SealedExportOffer, Vec) { + let body = b"scope v2\n"; + let content_sha256 = sha256_label(body); + let source_connection_id = SourceConnectionId::new("source-notion"); + let projection_id = ProjectionId::new("projection-readme"); + let records = vec![ + CanonicalExportRecord::File { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: None, + logical_path: LogicalPath::new("readme.md").expect("logical path"), + projection_id: projection_id.clone(), + }, + source_connection_id: source_connection_id.clone(), + file_kind: ProjectionFileKind::Markdown, + effective_actions: BTreeSet::from([SourceAction::Read]), + content_sha256: content_sha256.clone(), + byte_length: body.len() as u64, + }, + CanonicalExportRecord::Control { + order_key: CanonicalControlOrderKey { ordinal: 0 }, + member_path: locality_protocol::RESERVED_EXPORT_METADATA_PATH.to_string(), + }, + ]; + let source_generations = vec![OrderedSourceGeneration { + ordinal: 0, + source_connection_id: source_connection_id.clone(), + source_generation_id: SourceGenerationId::new("generation-scope").expect("generation ID"), + }]; + let limits = ExportAttemptLimits { + max_files: ReplicaMaterializationLimits::default() + .max_entries + .saturating_sub(1), + max_directories: ReplicaMaterializationLimits::default() + .max_entries + .saturating_sub(1), + max_content_bytes: ReplicaMaterializationLimits::default().max_disk_bytes, + }; + let inventory_sha256 = canonical_export_inventory_sha256(&records).expect("scope inventory"); + let writable_metadata = ScopeAuthorizedWritableExportMetadata { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: SessionId::new("session-7"), + export_attempt_id: ExportAttemptId::new("attempt-scope").expect("attempt ID"), + source_generations: source_generations.clone(), + writable_entries: Vec::new(), + }; + let writable_metadata_sha256 = + canonical_writable_metadata_sha256(&writable_metadata).expect("writable metadata digest"); + let offer = SealedExportOffer { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: writable_metadata.session_id.clone(), + export_attempt_id: writable_metadata.export_attempt_id.clone(), + source_generations: source_generations.clone(), + media_type: "application/x-tar".to_string(), + content_encoding: encoding, + limits, + control_entry_count: 1, + file_count: 1, + directory_count: 0, + archive_entry_count: 2, + selected_content_bytes: body.len() as u64, + inventory_sha256: inventory_sha256.clone(), + writable_metadata_sha256: writable_metadata_sha256.clone(), + sealed_at: "2026-07-23T20:00:01Z".to_string(), + expires_at: "2026-07-23T20:10:01Z".to_string(), + }; + let mut body_digest = DeliveredBodyDigestV2::new(1); + body_digest + .update_file(&projection_id, body) + .expect("update delivered-body digest"); + let receipt = ExportCompletionReceipt { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: offer.session_id.clone(), + export_attempt_id: offer.export_attempt_id.clone(), + source_generations, + inventory_sha256, + writable_metadata_sha256, + delivered_control_entry_count: 1, + delivered_file_count: 1, + delivered_directory_count: 0, + delivered_archive_entry_count: 2, + delivered_content_bytes: body.len() as u64, + delivered_body_sha256: body_digest.finish().expect("delivered-body digest"), + completed_at: "2026-07-23T20:00:03Z".to_string(), + }; + let terminal_control = ExportTerminalControlV2 { + writable_metadata, + completion_receipt: receipt, + }; + let mut builder = Builder::new(Vec::new()); + let pax = [ + (PAX_SOURCE_CONNECTION_ID, source_connection_id.as_str()), + (PAX_PROJECTION_ID, projection_id.as_str()), + (PAX_WINNING_SCOPE_ORDINAL, "0"), + (PAX_FILE_KIND, "markdown"), + (PAX_EFFECTIVE_ACTIONS, "[\"read\"]"), + (PAX_CONTENT_SHA256, content_sha256.as_str()), + ]; + builder + .append_pax_extensions(pax.map(|(key, value)| (key, value.as_bytes()))) + .expect("append scope PAX metadata"); + append_tar_member(&mut builder, b"readme.md", body); + append_tar_member( + &mut builder, + locality_protocol::RESERVED_EXPORT_METADATA_PATH.as_bytes(), + &serde_json::to_vec(&terminal_control).expect("serialize scope terminal control"), + ); + builder.finish().expect("finish scope tar"); + (offer, builder.into_inner().expect("collect scope tar")) +} + +fn append_tar_member(builder: &mut Builder>, path: &[u8], body: &[u8]) { + assert!(path.len() <= 100); + let mut header = Header::new_gnu(); + header.set_entry_type(EntryType::file()); + header.set_mode(0o444); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_size(body.len() as u64); + { + let bytes = header.as_mut_bytes(); + bytes[..100].fill(0); + bytes[..path.len()].copy_from_slice(path); + } + header.set_cksum(); + builder.append(&header, body).expect("append tar member"); +} + fn sha256_label(bytes: &[u8]) -> String { let digest = Sha256::digest(bytes); let hex = digest diff --git a/crates/localityd/src/replica_materializer.rs b/crates/localityd/src/replica_materializer.rs index a54d1b77..5ae0256c 100644 --- a/crates/localityd/src/replica_materializer.rs +++ b/crates/localityd/src/replica_materializer.rs @@ -27,6 +27,11 @@ use rustix::fs::RenameFlags; use rustix::fs::{AtFlags, Dir, FileType, Mode, OFlags, Stat}; use locality_core::portable::LogicalPath; +use locality_protocol::{ + CanonicalControlOrderKey, CanonicalDirectoryOrderKey, CanonicalExportRecord, + CanonicalFileOrderKey, DeliveredBodyDigestV2, ExportTerminalControlV2, ExportV2FilePaxMetadata, + MAX_EXPORT_TERMINAL_CONTROL_BYTES, SealedExportOffer, +}; use sha2::{Digest, Sha256}; use unicode_normalization::UnicodeNormalization; @@ -115,6 +120,7 @@ pub enum ReplicaMaterializationError { expected: u64, actual: u64, }, + ScopeExport(String), NonUtf8Path, InvalidPath { path: String, @@ -219,6 +225,12 @@ impl Display for ReplicaMaterializationError { formatter, "replica entry-count receipt mismatch: expected {expected}, actual {actual}" ), + Self::ScopeExport(reason) => { + write!( + formatter, + "invalid scope-authorized replica stream: {reason}" + ) + } Self::NonUtf8Path => formatter.write_str("replica tar entry path is not valid UTF-8"), Self::InvalidPath { path, reason } => { write!(formatter, "invalid replica path `{path}`: {reason}") @@ -306,6 +318,24 @@ pub fn materialize_replica_archive_with_expected_receipt( materialize_replica_archive_inner(archive, destination, limits, Some(expected)) } +/// Validate and atomically publish one scope-authorized v2 replica archive. +/// +/// V2 directories and files are consumed in canonical order, followed by one +/// hidden `.loc/session.json` completion receipt. File PAX metadata is used +/// only while recomputing the inventory and delivered-body digests; it is not +/// written into the published tree. +pub fn materialize_scope_authorized_replica_archive( + archive: ReplicaArchive, + destination: &Path, + limits: ReplicaMaterializationLimits, + offer: &SealedExportOffer, +) -> Result { + offer + .validate() + .map_err(|error| ReplicaMaterializationError::ScopeExport(error.to_string()))?; + materialize_scope_authorized_replica_archive_inner(archive, destination, limits, offer) +} + fn materialize_replica_archive_inner( archive: ReplicaArchive, destination: &Path, @@ -400,6 +430,95 @@ fn materialize_replica_archive_inner( Ok(summary) } +fn materialize_scope_authorized_replica_archive_inner( + archive: ReplicaArchive, + destination: &Path, + limits: ReplicaMaterializationLimits, + offer: &SealedExportOffer, +) -> Result { + let parent = destination + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .ok_or(ReplicaMaterializationError::InvalidDestination)?; + if destination.file_name().is_none() { + return Err(ReplicaMaterializationError::InvalidDestination); + } + match fs::symlink_metadata(parent) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) | Err(_) => { + return Err(ReplicaMaterializationError::DestinationParentMissing( + parent.to_path_buf(), + )); + } + } + if fs::symlink_metadata(destination).is_ok() { + return Err(ReplicaMaterializationError::DestinationExists( + destination.to_path_buf(), + )); + } + + let mut staging = StagingDirectory::create(parent)?; + let mut summary = match archive.encoding { + ReplicaArchiveEncoding::Identity => { + let mut decoded = DecodedLimitReader::new(archive.body, limits.max_decoded_bytes); + let result = extract_scope_authorized_tar(&mut decoded, &staging, limits, offer); + let exceeded = decoded.exceeded(); + let decoded_bytes = decoded.consumed(); + if exceeded { + return Err(ReplicaMaterializationError::DecodedLimit { + limit: limits.max_decoded_bytes, + }); + } + let mut summary = result?; + summary.decoded_bytes = decoded_bytes; + summary + } + ReplicaArchiveEncoding::Zstd => { + let mut decoder = zstd::stream::read::Decoder::new(archive.body) + .map_err(|error| ReplicaMaterializationError::Decode(error.to_string()))?; + decoder + .window_log_max(limits.max_zstd_window_log) + .map_err(|error| ReplicaMaterializationError::Decode(error.to_string()))?; + let mut decoder = decoder.single_frame(); + let (result, exceeded, decoded_bytes) = { + let mut decoded = DecodedLimitReader::new(&mut decoder, limits.max_decoded_bytes); + let result = extract_scope_authorized_tar(&mut decoded, &staging, limits, offer); + (result, decoded.exceeded(), decoded.consumed()) + }; + if exceeded { + return Err(ReplicaMaterializationError::DecodedLimit { + limit: limits.max_decoded_bytes, + }); + } + let mut summary = result?; + let mut compressed = decoder.finish(); + if read_one(&mut compressed) + .map_err(|error| ReplicaMaterializationError::Decode(error.to_string()))? + .is_some() + { + return Err(ReplicaMaterializationError::TrailingZstdData); + } + summary.decoded_bytes = decoded_bytes; + summary + } + }; + + make_tree_read_only(&staging).map_err(|source| ReplicaMaterializationError::Write { + path: staging.path().to_path_buf(), + source, + })?; + if fs::symlink_metadata(destination).is_ok() { + return Err(ReplicaMaterializationError::DestinationExists( + destination.to_path_buf(), + )); + } + staging.publish(destination)?; + // The hidden control member is an archive entry, but not a materialized + // file or byte. `extract_scope_authorized_tar` already counted it here. + summary.entries = offer.archive_entry_count; + Ok(summary) +} + fn validate_receipt( expected: ExpectedReplicaMaterializationReceipt, actual: ReplicaMaterializationSummary, @@ -520,6 +639,274 @@ fn extract_tar( Ok(state.summary) } +fn extract_scope_authorized_tar( + reader: &mut R, + staging: &StagingDirectory, + limits: ReplicaMaterializationLimits, + offer: &SealedExportOffer, +) -> Result { + let mut state = ExtractionState::default(); + let mut records = Vec::new(); + let mut terminal_control = None; + let mut body_digest = DeliveredBodyDigestV2::new(offer.file_count); + { + let mut archive = tar::Archive::new(reader.by_ref()); + let entries = archive + .entries() + .map_err(|error| ReplicaMaterializationError::MalformedTar(error.to_string()))?; + for entry in entries { + let mut entry = entry + .map_err(|error| ReplicaMaterializationError::MalformedTar(error.to_string()))?; + state.summary.entries = state.summary.entries.saturating_add(1); + if state.summary.entries > limits.max_entries { + return Err(ReplicaMaterializationError::EntryLimit { + limit: limits.max_entries, + }); + } + if terminal_control.is_some() { + return Err(scope_export( + "the completion receipt is not the final member", + )); + } + + let entry_type = entry.header().entry_type(); + let is_directory = entry_type.is_dir(); + if !entry_type.is_file() && !is_directory { + return Err(scope_export("an archive member has an unsupported type")); + } + if entry.header().link_name_bytes().is_some() { + return Err(scope_export("an archive member contains link metadata")); + } + + let raw_path = std::str::from_utf8(entry.path_bytes().as_ref()) + .map_err(|_| ReplicaMaterializationError::NonUtf8Path)? + .to_string(); + let normalized_path = if is_directory { + raw_path.strip_suffix('/').unwrap_or(&raw_path) + } else { + &raw_path + } + .to_string(); + let pax = locality_pax_fields(&mut entry)?; + + if normalized_path == locality_protocol::RESERVED_EXPORT_METADATA_PATH { + if !entry_type.is_file() { + return Err(scope_export("the completion receipt is not a regular file")); + } + if !pax.is_empty() { + return Err(scope_export( + "the completion receipt has locality PAX metadata", + )); + } + let mode = entry.header().mode().map_err(|error| { + ReplicaMaterializationError::MalformedTar(error.to_string()) + })?; + if mode != READ_ONLY_FILE_MODE { + return Err(scope_export("the completion receipt mode is not 0444")); + } + if entry.size() > MAX_EXPORT_TERMINAL_CONTROL_BYTES as u64 { + return Err(scope_export("the completion receipt is too large")); + } + let mut raw_control = Vec::with_capacity(entry.size() as usize); + entry + .read_to_end(&mut raw_control) + .map_err(|_| scope_export("the completion receipt is malformed"))?; + let parsed: ExportTerminalControlV2 = serde_json::from_slice(&raw_control) + .map_err(|_| scope_export("the completion receipt is malformed"))?; + if serde_json::to_vec(&parsed).ok().as_deref() != Some(raw_control.as_slice()) { + return Err(scope_export("the completion receipt is not canonical JSON")); + } + records.push(CanonicalExportRecord::Control { + order_key: CanonicalControlOrderKey { ordinal: 0 }, + member_path: locality_protocol::RESERVED_EXPORT_METADATA_PATH.to_string(), + }); + terminal_control = Some(parsed); + continue; + } + + let path = validated_path(entry.path_bytes().as_ref(), is_directory)?; + state.register_path(&path, is_directory)?; + let mode = entry + .header() + .mode() + .map_err(|error| ReplicaMaterializationError::MalformedTar(error.to_string()))?; + if is_directory { + if path.eq_ignore_ascii_case(".loc") { + return Err(scope_export( + "the reserved .loc directory header is forbidden", + )); + } + if !pax.is_empty() { + return Err(scope_export("a directory member has locality PAX metadata")); + } + if mode != READ_ONLY_DIRECTORY_MODE { + return Err(ReplicaMaterializationError::InvalidDirectoryMode { path, mode }); + } + if entry.size() != 0 { + return Err(ReplicaMaterializationError::NonEmptyDirectory { path }); + } + staging.create_directory(&path)?; + records.push(CanonicalExportRecord::Directory { + order_key: CanonicalDirectoryOrderKey { + depth: path.split('/').count() as u32, + logical_path: LogicalPath::new(path).map_err(|error| { + scope_export(format!("a directory path is invalid: {error}")) + })?, + }, + }); + continue; + } + + if mode != READ_ONLY_FILE_MODE { + return Err(ReplicaMaterializationError::InvalidFileMode { path, mode }); + } + let size = entry.size(); + if size > limits.max_file_bytes { + return Err(ReplicaMaterializationError::FileLimit { + path, + size, + limit: limits.max_file_bytes, + }); + } + let disk_size = state.summary.materialized_bytes.saturating_add(size); + if disk_size > limits.max_disk_bytes { + return Err(ReplicaMaterializationError::DiskLimit { + size: disk_size, + limit: limits.max_disk_bytes, + }); + } + let metadata = ExportV2FilePaxMetadata::from_records(&pax) + .ok_or_else(|| scope_export("a file has invalid locality PAX metadata"))?; + let logical_path = LogicalPath::new(path.clone()) + .map_err(|error| scope_export(format!("a file path is invalid: {error}")))?; + let parent_path = logical_path + .as_str() + .rsplit_once('/') + .map(|(parent, _)| LogicalPath::new(parent)) + .transpose() + .map_err(|_| scope_export("a file parent path is invalid"))?; + body_digest + .begin_file(&metadata.projection_id, size) + .map_err(|error| scope_export(error.to_string()))?; + let mut hashing = FileBodyHashReader::new(&mut entry, &mut body_digest); + staging.write_file(&path, &mut hashing, size)?; + let actual_content_sha256 = hashing.finish(); + body_digest + .end_file() + .map_err(|error| scope_export(error.to_string()))?; + if actual_content_sha256 != metadata.content_sha256 { + return Err(scope_export( + "a file body does not match its content digest", + )); + } + records.push(CanonicalExportRecord::File { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: metadata.winning_scope_ordinal, + parent_path, + logical_path, + projection_id: metadata.projection_id, + }, + source_connection_id: metadata.source_connection_id, + file_kind: metadata.file_kind, + effective_actions: metadata.effective_actions, + content_sha256: metadata.content_sha256, + byte_length: size, + }); + state.summary.files += 1; + state.summary.materialized_bytes = disk_size; + } + } + + let mut end_block = [0_u8; TAR_BLOCK_BYTES]; + if reader.read_exact(&mut end_block).is_err() || end_block.iter().any(|byte| *byte != 0) { + return Err(ReplicaMaterializationError::MissingTarEndMarker); + } + if read_one(reader) + .map_err(|error| ReplicaMaterializationError::MalformedTar(error.to_string()))? + .is_some() + { + return Err(ReplicaMaterializationError::TrailingTarData); + } + + let terminal_control = + terminal_control.ok_or_else(|| scope_export("the completion receipt is missing"))?; + terminal_control + .validate_against_inventory(offer, &records) + .map_err(|error| scope_export(error.to_string()))?; + let delivered_body_sha256 = body_digest + .finish() + .map_err(|error| scope_export(error.to_string()))?; + if terminal_control.completion_receipt.delivered_body_sha256 != delivered_body_sha256 { + return Err(scope_export( + "the delivered-body digest does not match the completion receipt", + )); + } + + state.summary.directories = state.filesystem_directories.len() as u64; + Ok(state.summary) +} + +fn scope_export(reason: impl Into) -> ReplicaMaterializationError { + ReplicaMaterializationError::ScopeExport(reason.into()) +} + +struct FileBodyHashReader<'a, R> { + inner: &'a mut R, + content: Sha256, + delivered: &'a mut DeliveredBodyDigestV2, +} + +impl<'a, R> FileBodyHashReader<'a, R> { + fn new(inner: &'a mut R, delivered: &'a mut DeliveredBodyDigestV2) -> Self { + Self { + inner, + content: Sha256::new(), + delivered, + } + } + + fn finish(self) -> String { + format!("sha256:{:x}", self.content.finalize()) + } +} + +impl Read for FileBodyHashReader<'_, R> { + fn read(&mut self, output: &mut [u8]) -> io::Result { + let read = self.inner.read(output)?; + self.content.update(&output[..read]); + self.delivered + .update_file_chunk(&output[..read]) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid delivered body"))?; + Ok(read) + } +} + +fn locality_pax_fields( + entry: &mut tar::Entry<'_, R>, +) -> Result, ReplicaMaterializationError> { + let mut fields = Vec::new(); + let Some(extensions) = entry + .pax_extensions() + .map_err(|_| scope_export("PAX metadata is malformed"))? + else { + return Ok(fields); + }; + for extension in extensions { + let extension = extension.map_err(|_| scope_export("PAX metadata is malformed"))?; + let key_bytes = extension.key_bytes(); + if !key_bytes.starts_with(b"locality.") { + continue; + } + let key = std::str::from_utf8(key_bytes) + .map_err(|_| scope_export("a locality PAX key is not UTF-8"))?; + let value = extension + .value() + .map_err(|_| scope_export("a locality PAX value is not UTF-8"))?; + fields.push((key.to_string(), value.to_string())); + } + Ok(fields) +} + fn validated_path( raw_path: &[u8], is_directory: bool, diff --git a/crates/localityd/tests/replica_materializer.rs b/crates/localityd/tests/replica_materializer.rs index ffceef92..832db767 100644 --- a/crates/localityd/tests/replica_materializer.rs +++ b/crates/localityd/tests/replica_materializer.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::fs; use std::io::{Cursor, Read}; use std::path::{Path, PathBuf}; @@ -7,11 +8,24 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::{AtomicU64, Ordering}; +use locality_core::portable::{ + ExportAttemptId, LogicalPath, ProjectionFileKind, ProjectionId, SessionId, SourceAction, + SourceConnectionId, SourceGenerationId, +}; +use locality_protocol::{ + CanonicalControlOrderKey, CanonicalDirectoryOrderKey, CanonicalExportRecord, + CanonicalFileOrderKey, DeliveredBodyDigestV2, ExportAttemptLimits, ExportCompletionReceipt, + ExportTerminalControlV2, OrderedSourceGeneration, PAX_CONTENT_SHA256, PAX_EFFECTIVE_ACTIONS, + PAX_FILE_KIND, PAX_PROJECTION_ID, PAX_SOURCE_CONNECTION_ID, PAX_WINNING_SCOPE_ORDINAL, + SCOPE_AUTHORIZED_COMPONENT_VERSIONS, ScopeAuthorizedWritableExportMetadata, SealedExportOffer, + TarContentEncoding, canonical_export_inventory_sha256, canonical_writable_metadata_sha256, +}; use localityd::remote_truth::{ReplicaArchive, ReplicaArchiveEncoding}; use localityd::replica_materializer::{ ExpectedReplicaMaterializationReceipt, ReplicaMaterializationLimits, ReplicaMaterializationSummary, materialize_replica_archive, materialize_replica_archive_with_expected_receipt, + materialize_scope_authorized_replica_archive, }; use sha2::{Digest, Sha256}; use tar::{Builder, EntryType, Header}; @@ -211,6 +225,352 @@ fn exact_receipt_identity_archive_publishes_after_decoded_tar_verification() { ); } +#[derive(Clone, Copy)] +enum V2ArchiveMutation { + None, + MissingPax, + DuplicatePax, + UnknownPax, + LocDirectoryHeader, + MissingReceipt, + MalformedReceipt, + NoncanonicalReceipt, + TrailingReceipt, + UnknownReceiptField, + DuplicateReceipt, + ReceiptNotFinal, + ReceiptBodyDigestMismatch, + ReceiptGenerationMismatch, + ReceiptCountMismatch, +} + +fn v2_offer_and_archive(mutation: V2ArchiveMutation) -> (SealedExportOffer, Vec) { + let body = b"scope authorized\n"; + let content_sha256 = sha256_label(Sha256::digest(body).into()); + let source_connection_id = SourceConnectionId::new("source-notion"); + let projection_id = ProjectionId::new("projection-readme"); + let actions = BTreeSet::from([SourceAction::Read, SourceAction::Search]); + let records = vec![ + CanonicalExportRecord::Directory { + order_key: CanonicalDirectoryOrderKey { + depth: 1, + logical_path: LogicalPath::new("docs").expect("directory path"), + }, + }, + CanonicalExportRecord::File { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: Some(LogicalPath::new("docs").expect("parent path")), + logical_path: LogicalPath::new("docs/readme.md").expect("file path"), + projection_id: projection_id.clone(), + }, + source_connection_id: source_connection_id.clone(), + file_kind: ProjectionFileKind::Markdown, + effective_actions: actions, + content_sha256: content_sha256.clone(), + byte_length: body.len() as u64, + }, + CanonicalExportRecord::Control { + order_key: CanonicalControlOrderKey { ordinal: 0 }, + member_path: locality_protocol::RESERVED_EXPORT_METADATA_PATH.to_string(), + }, + ]; + let source_generations = vec![OrderedSourceGeneration { + ordinal: 0, + source_connection_id: source_connection_id.clone(), + source_generation_id: SourceGenerationId::new("generation-9").expect("generation ID"), + }]; + let inventory_sha256 = + canonical_export_inventory_sha256(&records).expect("canonical inventory"); + let writable_metadata = ScopeAuthorizedWritableExportMetadata { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: SessionId::new("session-scope"), + export_attempt_id: ExportAttemptId::new("attempt-9").expect("attempt ID"), + source_generations: source_generations.clone(), + writable_entries: Vec::new(), + }; + let writable_metadata_sha256 = + canonical_writable_metadata_sha256(&writable_metadata).expect("writable metadata digest"); + let offer = SealedExportOffer { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: writable_metadata.session_id.clone(), + export_attempt_id: writable_metadata.export_attempt_id.clone(), + source_generations: source_generations.clone(), + media_type: "application/x-tar".to_string(), + content_encoding: TarContentEncoding::Identity, + limits: ExportAttemptLimits { + max_files: 10, + max_directories: 10, + max_content_bytes: 1024, + }, + control_entry_count: 1, + file_count: 1, + directory_count: 1, + archive_entry_count: 3, + selected_content_bytes: body.len() as u64, + inventory_sha256: inventory_sha256.clone(), + writable_metadata_sha256: writable_metadata_sha256.clone(), + sealed_at: "2026-07-23T20:00:00Z".to_string(), + expires_at: "2026-07-23T20:10:00Z".to_string(), + }; + offer.validate_inventory(&records).expect("valid offer"); + + let mut body_digest = DeliveredBodyDigestV2::new(1); + body_digest + .update_file(&projection_id, body) + .expect("body digest update"); + let delivered_body_sha256 = body_digest.finish().expect("body digest"); + let mut receipt = ExportCompletionReceipt { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: offer.session_id.clone(), + export_attempt_id: offer.export_attempt_id.clone(), + source_generations, + inventory_sha256, + writable_metadata_sha256, + delivered_control_entry_count: 1, + delivered_file_count: 1, + delivered_directory_count: 1, + delivered_archive_entry_count: 3, + delivered_content_bytes: body.len() as u64, + delivered_body_sha256, + completed_at: "2026-07-23T20:00:03Z".to_string(), + }; + if matches!(mutation, V2ArchiveMutation::ReceiptBodyDigestMismatch) { + receipt.delivered_body_sha256 = format!("sha256:{}", "0".repeat(64)); + } + if matches!(mutation, V2ArchiveMutation::ReceiptGenerationMismatch) { + receipt.source_generations[0].source_generation_id = + SourceGenerationId::new("generation-other").expect("generation ID"); + } + if matches!(mutation, V2ArchiveMutation::ReceiptCountMismatch) { + receipt.delivered_file_count = 2; + receipt.delivered_archive_entry_count = 4; + } + + let mut builder = Builder::new(Vec::new()); + append_test_member(&mut builder, &TestMember::directory("docs/")); + if matches!(mutation, V2ArchiveMutation::LocDirectoryHeader) { + append_test_member(&mut builder, &TestMember::directory(".loc/")); + } + let mut pax = vec![ + (PAX_SOURCE_CONNECTION_ID, source_connection_id.as_str()), + (PAX_PROJECTION_ID, projection_id.as_str()), + (PAX_WINNING_SCOPE_ORDINAL, "0"), + (PAX_FILE_KIND, "markdown"), + (PAX_EFFECTIVE_ACTIONS, "[\"read\",\"search\"]"), + (PAX_CONTENT_SHA256, content_sha256.as_str()), + ]; + match mutation { + V2ArchiveMutation::MissingPax => { + pax.retain(|(key, _)| *key != PAX_PROJECTION_ID); + } + V2ArchiveMutation::DuplicatePax => pax.push((PAX_PROJECTION_ID, "projection-other")), + V2ArchiveMutation::UnknownPax => pax.push(("locality.unknown", "forbidden")), + _ => {} + } + builder + .append_pax_extensions(pax.iter().map(|(key, value)| (*key, value.as_bytes()))) + .expect("append file PAX metadata"); + append_test_member(&mut builder, &TestMember::file("docs/readme.md", body)); + let terminal_control = ExportTerminalControlV2 { + writable_metadata, + completion_receipt: receipt, + }; + let mut receipt_body = + serde_json::to_vec(&terminal_control).expect("serialize terminal control"); + match mutation { + V2ArchiveMutation::MalformedReceipt => receipt_body = b"not-json".to_vec(), + V2ArchiveMutation::NoncanonicalReceipt => receipt_body.insert(0, b' '), + V2ArchiveMutation::TrailingReceipt => receipt_body.push(b'\n'), + V2ArchiveMutation::UnknownReceiptField => { + receipt_body.splice(1..1, b"\"unknown\":true,".iter().copied()); + } + _ => {} + } + let receipt_member = TestMember::file( + locality_protocol::RESERVED_EXPORT_METADATA_PATH, + receipt_body, + ); + if matches!(mutation, V2ArchiveMutation::ReceiptNotFinal) { + append_test_member(&mut builder, &receipt_member); + append_test_member(&mut builder, &TestMember::file("after.txt", "after")); + } else if !matches!(mutation, V2ArchiveMutation::MissingReceipt) { + append_test_member(&mut builder, &receipt_member); + if matches!(mutation, V2ArchiveMutation::DuplicateReceipt) { + append_test_member(&mut builder, &receipt_member); + } + } + builder.finish().expect("finish v2 tar"); + (offer, builder.into_inner().expect("collect v2 tar")) +} + +fn append_test_member(builder: &mut Builder>, member: &TestMember) { + let mut header = Header::new_gnu(); + header.set_entry_type(member.entry_type); + header.set_mode(member.mode); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_size(member.data.len() as u64); + assert!(member.path.len() <= 100, "test path fits GNU name field"); + { + let bytes = header.as_mut_bytes(); + bytes[..100].fill(0); + bytes[..member.path.len()].copy_from_slice(&member.path); + } + header.set_cksum(); + builder + .append(&header, member.data.as_slice()) + .expect("append test member"); +} + +fn materialize_v2( + archive: Vec, + encoding: ReplicaArchiveEncoding, + offer: &SealedExportOffer, + destination: &Path, +) -> Result { + materialize_scope_authorized_replica_archive( + ReplicaArchive::new(encoding, Cursor::new(archive)), + destination, + ReplicaMaterializationLimits::default(), + offer, + ) + .map_err(|error| error.to_string()) +} + +#[test] +fn scope_authorized_identity_and_zstd_publish_without_exposing_receipt() { + for (label, encoding) in [ + ("v2-identity", ReplicaArchiveEncoding::Identity), + ("v2-zstd", ReplicaArchiveEncoding::Zstd), + ] { + let root = TestDirectory::new(label); + let (mut offer, tar) = v2_offer_and_archive(V2ArchiveMutation::None); + let archive = match encoding { + ReplicaArchiveEncoding::Identity => tar, + ReplicaArchiveEncoding::Zstd => { + offer.content_encoding = TarContentEncoding::Zstd; + zstd::stream::encode_all(tar.as_slice(), 1).expect("compress v2 tar") + } + }; + let summary = materialize_v2(archive, encoding, &offer, &root.destination()) + .expect("materialize v2 archive"); + assert_eq!(summary.entries, 3); + assert_eq!(summary.files, 1); + assert_eq!(summary.directories, 1); + assert_eq!(summary.materialized_bytes, 17); + assert_eq!( + fs::read(root.destination().join("docs/readme.md")).expect("read v2 file"), + b"scope authorized\n" + ); + assert!(!root.destination().join(".loc").exists()); + } +} + +#[test] +fn scope_authorized_malformed_metadata_receipts_and_order_roll_back() { + for (label, mutation, expected) in [ + ( + "v2-missing-pax", + V2ArchiveMutation::MissingPax, + "invalid locality PAX metadata", + ), + ( + "v2-duplicate-pax", + V2ArchiveMutation::DuplicatePax, + "invalid locality PAX metadata", + ), + ( + "v2-unknown-pax", + V2ArchiveMutation::UnknownPax, + "invalid locality PAX metadata", + ), + ( + "v2-loc-directory-header", + V2ArchiveMutation::LocDirectoryHeader, + "reserved .loc directory header is forbidden", + ), + ( + "v2-missing-receipt", + V2ArchiveMutation::MissingReceipt, + "completion receipt is missing", + ), + ( + "v2-malformed-receipt", + V2ArchiveMutation::MalformedReceipt, + "completion receipt is malformed", + ), + ( + "v2-noncanonical-receipt", + V2ArchiveMutation::NoncanonicalReceipt, + "completion receipt is not canonical JSON", + ), + ( + "v2-trailing-receipt", + V2ArchiveMutation::TrailingReceipt, + "completion receipt is not canonical JSON", + ), + ( + "v2-unknown-receipt-field", + V2ArchiveMutation::UnknownReceiptField, + "completion receipt is not canonical JSON", + ), + ( + "v2-duplicate-receipt", + V2ArchiveMutation::DuplicateReceipt, + "completion receipt is not the final member", + ), + ( + "v2-receipt-not-final", + V2ArchiveMutation::ReceiptNotFinal, + "completion receipt is not the final member", + ), + ( + "v2-body-digest-mismatch", + V2ArchiveMutation::ReceiptBodyDigestMismatch, + "delivered-body digest does not match", + ), + ( + "v2-generation-mismatch", + V2ArchiveMutation::ReceiptGenerationMismatch, + "completion receipt does not match sealed export offer", + ), + ( + "v2-count-mismatch", + V2ArchiveMutation::ReceiptCountMismatch, + "completion receipt does not match sealed export offer", + ), + ] { + let root = TestDirectory::new(label); + let (offer, archive) = v2_offer_and_archive(mutation); + let error = materialize_v2( + archive, + ReplicaArchiveEncoding::Identity, + &offer, + &root.destination(), + ) + .expect_err("malformed v2 archive must fail"); + assert!(error.contains(expected), "{label}: {error}"); + root.assert_no_staging_or_destination(); + } +} + +#[test] +fn scope_authorized_truncation_rolls_back() { + let root = TestDirectory::new("v2-truncated"); + let (offer, mut archive) = v2_offer_and_archive(V2ArchiveMutation::None); + archive.truncate(archive.len() - 700); + materialize_v2( + archive, + ReplicaArchiveEncoding::Identity, + &offer, + &root.destination(), + ) + .expect_err("truncated v2 archive must fail"); + root.assert_no_staging_or_destination(); +} + struct ChunkedReader { inner: R, chunk_size: usize, From f1d77a7af3aad1b8c7113690494dbc5442c41100 Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 16:40:25 -0700 Subject: [PATCH 09/14] Retry ambiguous export attempt creation --- crates/loc-cli/src/sandbox.rs | 162 +++++++++++++++++++++++++++++----- 1 file changed, 141 insertions(+), 21 deletions(-) diff --git a/crates/loc-cli/src/sandbox.rs b/crates/loc-cli/src/sandbox.rs index 317d7799..af34fe5d 100644 --- a/crates/loc-cli/src/sandbox.rs +++ b/crates/loc-cli/src/sandbox.rs @@ -43,6 +43,7 @@ const HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(30 * 60); /// without imposing a 60-second total limit on a progressing export. const HTTP_READ_TIMEOUT: Duration = Duration::from_secs(60); const BOOTSTRAP_EXCHANGE_ATTEMPTS: usize = 2; +const EXPORT_ATTEMPT_CREATION_ATTEMPTS: usize = 2; const BOOTSTRAP_IDEMPOTENCY_DOMAIN: &[u8] = b"locality.session-exchange-idempotency.v1\0"; const IDEMPOTENCY_KEY_HEADER: &str = "Idempotency-Key"; const EXPORT_READ_AHEAD_CHUNK_BYTES: usize = 64 * 1024; @@ -1141,20 +1142,23 @@ impl SandboxHttpClient { operation: "bootstrap exchange", detail: error.without_url().to_string(), }; - if has_retry_remaining(attempt) { + if has_retry_remaining(attempt, BOOTSTRAP_EXCHANGE_ATTEMPTS) { continue; } return Err(error); } }; - if is_retriable_bootstrap_status(response.status()) && has_retry_remaining(attempt) { + if is_retriable_idempotent_status(response.status()) + && has_retry_remaining(attempt, BOOTSTRAP_EXCHANGE_ATTEMPTS) + { continue; } match read_json_response(response, "bootstrap exchange") { Ok(capability) => return Ok(capability), Err(error) - if has_retry_remaining(attempt) && is_ambiguous_bootstrap_error(&error) => + if has_retry_remaining(attempt, BOOTSTRAP_EXCHANGE_ATTEMPTS) + && is_ambiguous_idempotent_error(&error, "bootstrap exchange") => { continue; } @@ -1225,18 +1229,46 @@ impl SandboxHttpClient { capability: &SessionCapability, request: &ExportAttemptRequest, ) -> Result { - let response = self - .client - .post(self.export_attempts_url(capability.session_id.as_str())) - .header(ACCEPT, JSON_MEDIA_TYPE) - .bearer_auth(&capability.opaque_capability) - .json(request) - .send() - .map_err(|error| SandboxInitError::Http { - operation: "export-attempt creation", - detail: error.without_url().to_string(), - })?; - read_json_response(response, "export-attempt creation") + for attempt in 0..EXPORT_ATTEMPT_CREATION_ATTEMPTS { + let response = match self + .client + .post(self.export_attempts_url(capability.session_id.as_str())) + .header(ACCEPT, JSON_MEDIA_TYPE) + .bearer_auth(&capability.opaque_capability) + .json(request) + .send() + { + Ok(response) => response, + Err(error) => { + let error = SandboxInitError::Http { + operation: "export-attempt creation", + detail: error.without_url().to_string(), + }; + if has_retry_remaining(attempt, EXPORT_ATTEMPT_CREATION_ATTEMPTS) { + continue; + } + return Err(error); + } + }; + + if is_retriable_idempotent_status(response.status()) + && has_retry_remaining(attempt, EXPORT_ATTEMPT_CREATION_ATTEMPTS) + { + continue; + } + match read_json_response(response, "export-attempt creation") { + Ok(offer) => return Ok(offer), + Err(error) + if has_retry_remaining(attempt, EXPORT_ATTEMPT_CREATION_ATTEMPTS) + && is_ambiguous_idempotent_error(&error, "export-attempt creation") => + { + continue; + } + Err(error) => return Err(error), + } + } + + unreachable!("export-attempt creation loop always returns") } fn open_export_attempt( @@ -1332,24 +1364,24 @@ fn lower_hex(bytes: &[u8]) -> String { String::from_utf8(encoded).expect("lowercase hexadecimal is valid UTF-8") } -fn has_retry_remaining(attempt: usize) -> bool { - attempt + 1 < BOOTSTRAP_EXCHANGE_ATTEMPTS +fn has_retry_remaining(attempt: usize, max_attempts: usize) -> bool { + attempt + 1 < max_attempts } -fn is_retriable_bootstrap_status(status: StatusCode) -> bool { +fn is_retriable_idempotent_status(status: StatusCode) -> bool { matches!( status, StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE | StatusCode::GATEWAY_TIMEOUT ) } -fn is_ambiguous_bootstrap_error(error: &SandboxInitError) -> bool { +fn is_ambiguous_idempotent_error(error: &SandboxInitError, operation: &'static str) -> bool { matches!( error, SandboxInitError::Http { - operation: "bootstrap exchange", + operation: actual_operation, .. - } + } if *actual_operation == operation ) } @@ -1880,6 +1912,80 @@ mod tests { assert_eq!(idempotency_key(&requests[0]), idempotency_key(&requests[1])); } + #[test] + fn dropped_export_attempt_response_retries_the_exact_sealed_request() { + let offer = scope_offer_fixture(); + let response = serde_json::to_vec(&offer).expect("serialize scope offer"); + let server = TestServer::start( + vec![ + TestResponse::DropConnection, + TestResponse::Json { + status: "200 OK", + body: response, + }, + ], + true, + ); + let client = SandboxHttpClient::new(&server.api_url).expect("HTTP client"); + let request = scope_attempt_request_fixture(); + + assert_eq!( + client + .create_export_attempt(&capability(), &request) + .expect("retry export-attempt creation"), + offer + ); + + let requests = server.finish(); + assert_eq!(requests.len(), EXPORT_ATTEMPT_CREATION_ATTEMPTS); + assert_eq!(requests[0].method, "POST"); + assert_eq!( + requests[0].path, + "/v1/sessions/session-idempotent/export-attempts" + ); + assert_eq!(requests[0].body, requests[1].body); + assert_eq!( + serde_json::from_slice::(&requests[0].body) + .expect("decode captured request") + .idempotency_key, + request.idempotency_key + ); + } + + #[test] + fn export_attempt_gateway_retry_is_bounded_and_reuses_the_request() { + let server = TestServer::start( + vec![ + TestResponse::Json { + status: "503 Service Unavailable", + body: Vec::new(), + }, + TestResponse::Json { + status: "503 Service Unavailable", + body: Vec::new(), + }, + ], + true, + ); + let client = SandboxHttpClient::new(&server.api_url).expect("HTTP client"); + let request = scope_attempt_request_fixture(); + + let error = client + .create_export_attempt(&capability(), &request) + .expect_err("repeated service failure must stop"); + + assert!(matches!( + error, + SandboxInitError::HttpStatus { + operation: "export-attempt creation", + status: StatusCode::SERVICE_UNAVAILABLE, + } + )); + let requests = server.finish(); + assert_eq!(requests.len(), EXPORT_ATTEMPT_CREATION_ATTEMPTS); + assert_eq!(requests[0].body, requests[1].body); + } + #[test] fn bootstrap_idempotency_keys_are_stable_per_token_and_separate_between_tokens() { let response = serde_json::to_vec(&capability()).expect("serialize capability"); @@ -2004,6 +2110,20 @@ mod tests { } } + fn scope_attempt_request_fixture() -> ExportAttemptRequest { + serde_json::from_str(include_str!( + "../../locality-protocol/fixtures/export-attempt-request.json" + )) + .expect("scope export-attempt request fixture") + } + + fn scope_offer_fixture() -> SealedExportOffer { + serde_json::from_str(include_str!( + "../../locality-protocol/fixtures/sealed-export-offer.json" + )) + .expect("sealed scope export offer fixture") + } + fn assert_bootstrap_request(request: &CapturedRequest, expected_body: &[u8]) { assert_eq!(request.method, "POST"); assert_eq!(request.path, "/v1/sessions"); From 188a521417b8c11e2cefdab4128f43af23d0afb0 Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 16:47:32 -0700 Subject: [PATCH 10/14] Canonicalize export completion receipt digests --- crates/locality-protocol/Cargo.toml | 4 +--- crates/locality-protocol/src/lib.rs | 14 ++++++++++++++ crates/locality-protocol/tests/golden.rs | 5 +++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/locality-protocol/Cargo.toml b/crates/locality-protocol/Cargo.toml index 02ea8bef..27eb9a3b 100644 --- a/crates/locality-protocol/Cargo.toml +++ b/crates/locality-protocol/Cargo.toml @@ -13,7 +13,5 @@ path = "src/lib.rs" [dependencies] locality-core.workspace = true serde = { version = "1.0", features = ["derive"] } -sha2 = "0.10" - -[dev-dependencies] serde_json = "1.0" +sha2 = "0.10" diff --git a/crates/locality-protocol/src/lib.rs b/crates/locality-protocol/src/lib.rs index 1ecbad99..db7ce482 100644 --- a/crates/locality-protocol/src/lib.rs +++ b/crates/locality-protocol/src/lib.rs @@ -1512,6 +1512,20 @@ impl ExportCompletionReceipt { } } +/// Domain-separated digest of the exact compact JSON completion receipt +/// persisted by the private delivery audit. The receipt contains no maps, so +/// its derived field order is the canonical JSON order for this version. +pub fn canonical_export_completion_receipt_sha256( + receipt: &ExportCompletionReceipt, +) -> Result { + receipt.validate()?; + let mut preimage = b"locality.export.completion-receipt.v2\0".to_vec(); + preimage.extend_from_slice( + &serde_json::to_vec(receipt).map_err(|_| ScopeContractError::InvalidControlRecord)?, + ); + Ok(format!("sha256:{:x}", Sha256::digest(preimage))) +} + /// Encodings and exact decoded bounds available for one immutable tar export. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct TarExportOffer { diff --git a/crates/locality-protocol/tests/golden.rs b/crates/locality-protocol/tests/golden.rs index 69ef143c..4182d016 100644 --- a/crates/locality-protocol/tests/golden.rs +++ b/crates/locality-protocol/tests/golden.rs @@ -91,6 +91,11 @@ fn scope_authorized_export_contracts_are_exact_golden_bytes() { .validate_against(&offer) .expect("completion matches sealed metadata selection"); assert_exact_round_trip(EXPORT_COMPLETION_RECEIPT_GOLDEN_JSON, &receipt); + assert_eq!( + locality_protocol::canonical_export_completion_receipt_sha256(&receipt) + .expect("completion receipt digest"), + "sha256:8e5e9d9e12fbb951e1127f97a2214d7462cb685973cdb6c3d8f4c540cec080ae" + ); let control = export_terminal_control_v2(); control From a6ef9058abc080a5e72686fd3ff855661ef6378c Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 17:16:08 -0700 Subject: [PATCH 11/14] Stress scope-authorized sandbox materialization --- crates/loc-cli/tests/sandbox.rs | 495 +++++++++++++++++- .../localityd/tests/replica_materializer.rs | 369 ++++++++++++- 2 files changed, 844 insertions(+), 20 deletions(-) diff --git a/crates/loc-cli/tests/sandbox.rs b/crates/loc-cli/tests/sandbox.rs index 448db2ab..8c85f8da 100644 --- a/crates/loc-cli/tests/sandbox.rs +++ b/crates/loc-cli/tests/sandbox.rs @@ -7,7 +7,7 @@ use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc::{self, Receiver}; use std::thread::{self, JoinHandle}; -use std::time::Duration; +use std::time::{Duration, Instant}; use loc_cli::sandbox::{ SandboxBootstrapToken, SandboxContentEncodingPreference, SandboxInitError, SandboxInitOptions, @@ -18,15 +18,15 @@ use locality_core::portable::{ SourceConnectionId, SourceGenerationId, }; use locality_protocol::{ - COMPONENT_VERSIONS, CanonicalControlOrderKey, CanonicalExportRecord, CanonicalFileOrderKey, - ComponentVersions, DeliveredBodyDigestV2, ExportAttemptLimits, ExportAttemptRequest, - ExportCompletionReceipt, ExportTerminalControlV2, FreshnessRequirement, - OpaqueBootstrapExchangeRequest, OrderedSourceGeneration, PAX_CONTENT_SHA256, - PAX_EFFECTIVE_ACTIONS, PAX_FILE_KIND, PAX_PROJECTION_ID, PAX_SOURCE_CONNECTION_ID, - PAX_WINNING_SCOPE_ORDINAL, SCOPE_AUTHORIZED_COMPONENT_VERSIONS, SandboxSessionState, - SandboxSessionStatus, ScopeAuthorizedWritableExportMetadata, SealedExportOffer, - SessionCapability, SessionErrorCode, SessionProtocolError, StaleSessionBehavior, - TarContentEncoding, TarExportOffer, canonical_export_inventory_sha256, + COMPONENT_VERSIONS, CanonicalControlOrderKey, CanonicalDirectoryOrderKey, + CanonicalExportRecord, CanonicalFileOrderKey, ComponentVersions, DeliveredBodyDigestV2, + ExportAttemptLimits, ExportAttemptRequest, ExportCompletionReceipt, ExportTerminalControlV2, + FreshnessRequirement, OpaqueBootstrapExchangeRequest, OrderedSourceGeneration, + PAX_CONTENT_SHA256, PAX_EFFECTIVE_ACTIONS, PAX_FILE_KIND, PAX_PROJECTION_ID, + PAX_SOURCE_CONNECTION_ID, PAX_WINNING_SCOPE_ORDINAL, SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + SandboxSessionState, SandboxSessionStatus, ScopeAuthorizedWritableExportMetadata, + SealedExportOffer, SessionCapability, SessionErrorCode, SessionProtocolError, + StaleSessionBehavior, TarContentEncoding, TarExportOffer, canonical_export_inventory_sha256, canonical_writable_metadata_sha256, }; use localityd::replica_materializer::ReplicaMaterializationLimits; @@ -74,6 +74,7 @@ struct ResponseFixture { body: Vec, declared_content_length: Option, split_after: Option<(usize, Duration)>, + body_chunk_size: Option, staging_gate: Option<(PathBuf, PathBuf, PathBuf)>, } @@ -85,6 +86,7 @@ impl ResponseFixture { body: serde_json::to_vec(value).expect("serialize response"), declared_content_length: None, split_after: None, + body_chunk_size: None, staging_gate: None, } } @@ -99,6 +101,7 @@ impl ResponseFixture { body, declared_content_length: None, split_after: None, + body_chunk_size: None, staging_gate: None, } } @@ -122,6 +125,12 @@ impl ResponseFixture { self } + fn with_body_chunk_size(mut self, bytes: usize) -> Self { + assert!(bytes > 0, "response chunk size must be positive"); + self.body_chunk_size = Some(bytes); + self + } + fn with_staging_gate( mut self, parent: PathBuf, @@ -416,6 +425,116 @@ fn scope_authorized_export_attempts_materialize_identity_and_zstd() { assert_ne!(idempotency_keys[0], idempotency_keys[1]); } +#[test] +fn scope_authorized_real_world_export_streams_through_the_client() { + let directory = TestDirectory::new("scope-real-world"); + let capability = capability(); + let status = scope_ready_status(capability.session_id.clone()); + let (offer, tar) = real_world_scope_export_fixture(TarContentEncoding::Zstd); + let compressed = zstd::stream::encode_all(tar.as_slice(), 1).expect("compress stress tar"); + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ResponseFixture::json(&offer), + ResponseFixture::export("zstd", compressed).with_body_chunk_size(4093), + ]); + + let started = Instant::now(); + let report = run_sandbox_init_with_encoding( + SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }, + SandboxBootstrapToken::new("scope-real-world-bootstrap").expect("token"), + SandboxContentEncodingPreference::Zstd, + ) + .expect("stream and materialize real-world scope export"); + eprintln!( + "scope real-world client materialization: {} files, {} directories, {} bytes in {:?}", + report.files, + report.directories, + report.materialized_bytes, + started.elapsed() + ); + + assert_eq!(report.entries, 1_282); + assert_eq!(report.files, 667); + assert_eq!(report.directories, 614); + assert_eq!(report.materialized_bytes, 47 * 1024 * 1024); + assert_eq!( + fs::read(directory.root().join("root-000/empty.bin")).expect("read zero-byte file"), + b"" + ); + assert_eq!( + fs::read( + directory + .root() + .join("root-000/d00/d01/d02/d03/d04/d05/d06/d07/d08/d09/d10/d11/d12/d13/page.md") + ) + .expect("read deep Markdown file"), + b"# General Teamspace\n\nReal-world scope fixture.\n" + ); + assert_eq!( + fs::metadata(directory.root().join("root-099/blob-0099.bin")) + .expect("stat repeated-parent binary") + .len(), + 74_110 + ); + assert!(!directory.root().join(".loc").exists()); + assert_test_mode(&directory.root(), 0o555); + assert_test_mode(&directory.root().join("root-000/empty.bin"), 0o444); + + let _ = server.request(); + let _ = server.request(); + let _ = server.request(); + let export = server.request(); + assert_eq!(export.headers.get("accept-encoding").unwrap(), "zstd"); +} + +#[test] +fn scope_authorized_repeat_initialization_reports_destination_conflict() { + let directory = TestDirectory::new("scope-repeat-destination"); + let capability = capability(); + let status = scope_ready_status(capability.session_id.clone()); + let (offer, tar) = scope_export_fixture(TarContentEncoding::Identity); + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ResponseFixture::json(&offer), + ResponseFixture::export("identity", tar), + ]); + let options = SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }; + + run_sandbox_init_with_encoding( + options.clone(), + SandboxBootstrapToken::new("scope-first-bootstrap").expect("token"), + SandboxContentEncodingPreference::Identity, + ) + .expect("first initialization publishes"); + let error = run_sandbox_init_with_encoding( + options, + SandboxBootstrapToken::new("scope-repeat-bootstrap").expect("token"), + SandboxContentEncodingPreference::Identity, + ) + .expect_err("repeat initialization must not replace existing publication"); + + assert_eq!(error.code(), "destination_invalid"); + assert!(error.to_string().contains("already exists")); + assert!(!error.to_string().contains("scope-repeat-bootstrap")); + assert_eq!( + fs::read(directory.root().join("readme.md")).expect("read first publication"), + b"scope v2\n" + ); + assert!(!directory.root().join(".loc").exists()); + for _ in 0..4 { + let _ = server.request(); + } + server.assert_no_request(); +} + #[test] fn malformed_scope_offer_fails_before_stream_without_identifier_leaks() { let directory = TestDirectory::new("scope-malformed-offer"); @@ -460,6 +579,38 @@ fn malformed_scope_offer_fails_before_stream_without_identifier_leaks() { server.assert_no_request(); } +#[test] +fn scope_authorized_extra_path_is_rejected_without_partial_publication() { + let directory = TestDirectory::new("scope-unauthorized-extra"); + let capability = capability(); + let status = scope_ready_status(capability.session_id.clone()); + let (offer, tar) = + scope_export_fixture_with_unauthorized_extra(TarContentEncoding::Identity, true); + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ResponseFixture::json(&offer), + ResponseFixture::export("identity", tar).with_body_chunk_size(73), + ]); + + let error = run_sandbox_init_with_encoding( + SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }, + SandboxBootstrapToken::new("scope-unauthorized-bootstrap").expect("token"), + SandboxContentEncodingPreference::Identity, + ) + .expect_err("an extra path outside the sealed inventory must fail"); + + assert_eq!(error.code(), "materialization_failed"); + assert!(!error.to_string().contains("scope-unauthorized-bootstrap")); + assert!(!directory.root().exists()); + for _ in 0..4 { + let _ = server.request(); + } +} + #[test] fn export_bytes_are_staged_while_the_http_response_is_still_streaming() { let directory = TestDirectory::new("streaming-overlap"); @@ -866,6 +1017,7 @@ fn bearer_authenticated_redirect_is_not_followed() { body: Vec::new(), declared_content_length: None, split_after: None, + body_chunk_size: None, staging_gate: None, }, ]); @@ -1040,6 +1192,7 @@ fn version_session_offer_media_and_response_encoding_are_validated() { body: tar.clone(), declared_content_length: None, split_after: None, + body_chunk_size: None, staging_gate: None, }), "backend_protocol_invalid", @@ -1422,6 +1575,13 @@ fn scope_ready_status(session_id: SessionId) -> SandboxSessionStatus { } fn scope_export_fixture(encoding: TarContentEncoding) -> (SealedExportOffer, Vec) { + scope_export_fixture_with_unauthorized_extra(encoding, false) +} + +fn scope_export_fixture_with_unauthorized_extra( + encoding: TarContentEncoding, + unauthorized_extra: bool, +) -> (SealedExportOffer, Vec) { let body = b"scope v2\n"; let content_sha256 = sha256_label(body); let source_connection_id = SourceConnectionId::new("source-notion"); @@ -1523,6 +1683,27 @@ fn scope_export_fixture(encoding: TarContentEncoding) -> (SealedExportOffer, Vec .append_pax_extensions(pax.map(|(key, value)| (key, value.as_bytes()))) .expect("append scope PAX metadata"); append_tar_member(&mut builder, b"readme.md", body); + if unauthorized_extra { + let extra_body = b"unauthorized\n"; + let extra_sha256 = sha256_label(extra_body); + let extra_projection_id = ProjectionId::new("projection-unauthorized"); + let extra_pax = [ + (PAX_SOURCE_CONNECTION_ID, source_connection_id.as_str()), + (PAX_PROJECTION_ID, extra_projection_id.as_str()), + (PAX_WINNING_SCOPE_ORDINAL, "0"), + (PAX_FILE_KIND, "markdown"), + (PAX_EFFECTIVE_ACTIONS, "[\"read\"]"), + (PAX_CONTENT_SHA256, extra_sha256.as_str()), + ]; + builder + .append_pax_extensions( + extra_pax + .iter() + .map(|(key, value)| (*key, value.as_bytes())), + ) + .expect("append unauthorized PAX metadata"); + append_tar_member(&mut builder, b"unauthorized.md", extra_body); + } append_tar_member( &mut builder, locality_protocol::RESERVED_EXPORT_METADATA_PATH.as_bytes(), @@ -1532,6 +1713,246 @@ fn scope_export_fixture(encoding: TarContentEncoding) -> (SealedExportOffer, Vec (offer, builder.into_inner().expect("collect scope tar")) } +#[derive(Clone)] +struct RealWorldFile { + order_key: CanonicalFileOrderKey, + file_kind: ProjectionFileKind, + byte_length: usize, + fill: u8, +} + +fn real_world_file_body(file: &RealWorldFile) -> Vec { + if file.order_key.logical_path.as_str().ends_with("page.md") { + b"# General Teamspace\n\nReal-world scope fixture.\n".to_vec() + } else { + vec![file.fill; file.byte_length] + } +} + +fn real_world_scope_export_fixture(encoding: TarContentEncoding) -> (SealedExportOffer, Vec) { + const DIRECTORY_COUNT: u64 = 614; + const FILE_COUNT: u64 = 667; + const CONTENT_BYTES: u64 = 47 * 1024 * 1024; + + let source_connection_id = SourceConnectionId::new("source-notion-real-world"); + let source_generations = vec![OrderedSourceGeneration { + ordinal: 0, + source_connection_id: source_connection_id.clone(), + source_generation_id: SourceGenerationId::new("generation-real-world") + .expect("generation ID"), + }]; + let mut directories = (0..600) + .map(|index| format!("root-{index:03}")) + .collect::>(); + let mut deep_path = "root-000".to_string(); + for depth in 0..14 { + deep_path.push_str(&format!("/d{depth:02}")); + directories.push(deep_path.clone()); + } + directories.sort_by_key(|path| CanonicalDirectoryOrderKey { + depth: path.split('/').count() as u32, + logical_path: LogicalPath::new(path).expect("directory path"), + }); + assert_eq!(directories.len() as u64, DIRECTORY_COUNT); + + let markdown = b"# General Teamspace\n\nReal-world scope fixture.\n"; + let binary_count = FILE_COUNT as usize - 2; + let binary_bytes = CONTENT_BYTES as usize - markdown.len(); + let base_binary_bytes = binary_bytes / binary_count; + let extra_binary_bytes = binary_bytes % binary_count; + let mut files = vec![ + RealWorldFile { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: Some(LogicalPath::new("root-000").expect("empty parent")), + logical_path: LogicalPath::new("root-000/empty.bin").expect("empty path"), + projection_id: ProjectionId::new("projection-empty"), + }, + file_kind: ProjectionFileKind::Binary, + byte_length: 0, + fill: 0, + }, + RealWorldFile { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: Some(LogicalPath::new(&deep_path).expect("deep parent")), + logical_path: LogicalPath::new(format!("{deep_path}/page.md")) + .expect("deep page path"), + projection_id: ProjectionId::new("projection-page"), + }, + file_kind: ProjectionFileKind::Markdown, + byte_length: markdown.len(), + fill: 0, + }, + ]; + for index in 0..binary_count { + let parent = format!("root-{:03}", index % 100); + files.push(RealWorldFile { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: Some(LogicalPath::new(&parent).expect("binary parent")), + logical_path: LogicalPath::new(format!("{parent}/blob-{index:04}.bin")) + .expect("binary path"), + projection_id: ProjectionId::new(format!("projection-blob-{index:04}")), + }, + file_kind: ProjectionFileKind::Binary, + byte_length: base_binary_bytes + usize::from(index < extra_binary_bytes), + fill: (index % 251) as u8, + }); + } + files.sort_by(|left, right| left.order_key.cmp(&right.order_key)); + assert_eq!(files.len() as u64, FILE_COUNT); + + let mut records = Vec::with_capacity((DIRECTORY_COUNT + FILE_COUNT + 1) as usize); + let mut builder = Builder::new(Vec::new()); + for path in &directories { + records.push(CanonicalExportRecord::Directory { + order_key: CanonicalDirectoryOrderKey { + depth: path.split('/').count() as u32, + logical_path: LogicalPath::new(path).expect("directory path"), + }, + }); + append_tar_directory(&mut builder, format!("{path}/").as_bytes()); + } + + let actions = BTreeSet::from([SourceAction::Read]); + let mut body_digest = DeliveredBodyDigestV2::new(FILE_COUNT); + let mut content_bytes = 0_u64; + for file in &files { + let body = real_world_file_body(file); + assert_eq!(body.len(), file.byte_length); + content_bytes += body.len() as u64; + let content_sha256 = sha256_label(&body); + body_digest + .update_file(&file.order_key.projection_id, &body) + .expect("update real-world body digest"); + records.push(CanonicalExportRecord::File { + order_key: file.order_key.clone(), + source_connection_id: source_connection_id.clone(), + file_kind: file.file_kind.clone(), + effective_actions: actions.clone(), + content_sha256: content_sha256.clone(), + byte_length: body.len() as u64, + }); + let file_kind = match file.file_kind { + ProjectionFileKind::Markdown => "markdown", + ProjectionFileKind::Binary => "binary", + _ => unreachable!("real-world fixture only uses Markdown and binary files"), + }; + let pax = [ + (PAX_SOURCE_CONNECTION_ID, source_connection_id.as_str()), + (PAX_PROJECTION_ID, file.order_key.projection_id.as_str()), + (PAX_WINNING_SCOPE_ORDINAL, "0"), + (PAX_FILE_KIND, file_kind), + (PAX_EFFECTIVE_ACTIONS, "[\"read\"]"), + (PAX_CONTENT_SHA256, content_sha256.as_str()), + ]; + builder + .append_pax_extensions(pax.iter().map(|(key, value)| (*key, value.as_bytes()))) + .expect("append real-world PAX metadata"); + append_tar_member( + &mut builder, + file.order_key.logical_path.as_str().as_bytes(), + &body, + ); + } + assert_eq!(content_bytes, CONTENT_BYTES); + records.push(CanonicalExportRecord::Control { + order_key: CanonicalControlOrderKey { ordinal: 0 }, + member_path: locality_protocol::RESERVED_EXPORT_METADATA_PATH.to_string(), + }); + + let writable_metadata = ScopeAuthorizedWritableExportMetadata { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: SessionId::new("session-7"), + export_attempt_id: ExportAttemptId::new("attempt-real-world").expect("attempt ID"), + source_generations: source_generations.clone(), + writable_entries: Vec::new(), + }; + let inventory_sha256 = + canonical_export_inventory_sha256(&records).expect("canonical real-world inventory"); + let writable_metadata_sha256 = + canonical_writable_metadata_sha256(&writable_metadata).expect("writable metadata digest"); + let archive_entry_count = DIRECTORY_COUNT + FILE_COUNT + 1; + let offer = SealedExportOffer { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: writable_metadata.session_id.clone(), + export_attempt_id: writable_metadata.export_attempt_id.clone(), + source_generations: source_generations.clone(), + media_type: "application/x-tar".to_string(), + content_encoding: encoding, + limits: ExportAttemptLimits { + max_files: ReplicaMaterializationLimits::default() + .max_entries + .saturating_sub(1), + max_directories: ReplicaMaterializationLimits::default() + .max_entries + .saturating_sub(1), + max_content_bytes: ReplicaMaterializationLimits::default().max_disk_bytes, + }, + control_entry_count: 1, + file_count: FILE_COUNT, + directory_count: DIRECTORY_COUNT, + archive_entry_count, + selected_content_bytes: CONTENT_BYTES, + inventory_sha256: inventory_sha256.clone(), + writable_metadata_sha256: writable_metadata_sha256.clone(), + sealed_at: "2026-07-23T20:00:01Z".to_string(), + expires_at: "2026-07-23T20:10:01Z".to_string(), + }; + offer + .validate_inventory(&records) + .expect("valid real-world inventory"); + let terminal_control = ExportTerminalControlV2 { + writable_metadata, + completion_receipt: ExportCompletionReceipt { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: offer.session_id.clone(), + export_attempt_id: offer.export_attempt_id.clone(), + source_generations, + inventory_sha256, + writable_metadata_sha256, + delivered_control_entry_count: 1, + delivered_file_count: FILE_COUNT, + delivered_directory_count: DIRECTORY_COUNT, + delivered_archive_entry_count: archive_entry_count, + delivered_content_bytes: CONTENT_BYTES, + delivered_body_sha256: body_digest.finish().expect("real-world body digest"), + completed_at: "2026-07-23T20:00:03Z".to_string(), + }, + }; + append_tar_member( + &mut builder, + locality_protocol::RESERVED_EXPORT_METADATA_PATH.as_bytes(), + &serde_json::to_vec(&terminal_control).expect("serialize terminal control"), + ); + builder.finish().expect("finish real-world scope tar"); + ( + offer, + builder.into_inner().expect("collect real-world scope tar"), + ) +} + +fn append_tar_directory(builder: &mut Builder>, path: &[u8]) { + assert!(path.len() <= 100); + let mut header = Header::new_gnu(); + header.set_entry_type(EntryType::dir()); + header.set_mode(0o555); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_size(0); + { + let bytes = header.as_mut_bytes(); + bytes[..100].fill(0); + bytes[..path.len()].copy_from_slice(path); + } + header.set_cksum(); + builder + .append(&header, std::io::empty()) + .expect("append tar directory"); +} + fn append_tar_member(builder: &mut Builder>, path: &[u8], body: &[u8]) { assert!(path.len() <= 100); let mut header = Header::new_gnu(); @@ -1665,9 +2086,11 @@ fn write_response(stream: &mut TcpStream, response: ResponseFixture) { write!(stream, "\r\n").expect("finish response headers"); if let Some((split_after, pause)) = response.split_after { let split_after = split_after.min(response.body.len()); - stream - .write_all(&response.body[..split_after]) - .expect("write first response body chunk"); + write_response_body( + stream, + &response.body[..split_after], + response.body_chunk_size, + ); stream.flush().expect("flush first response body chunk"); if let Some((parent, logical_path, destination)) = &response.staging_gate { let deadline = std::time::Instant::now() + Duration::from_secs(2); @@ -1698,23 +2121,57 @@ fn write_response(stream: &mut TcpStream, response: ResponseFixture) { } } thread::sleep(pause); - stream - .write_all(&response.body[split_after..]) - .expect("write remaining response body"); + write_response_body( + stream, + &response.body[split_after..], + response.body_chunk_size, + ); } else { - stream - .write_all(&response.body) - .expect("write response body"); + write_response_body(stream, &response.body, response.body_chunk_size); } stream.flush().expect("flush response"); } +fn write_response_body(stream: &mut TcpStream, body: &[u8], chunk_size: Option) { + let chunk_size = chunk_size.unwrap_or(body.len().max(1)); + for chunk in body.chunks(chunk_size) { + stream.write_all(chunk).expect("write response body chunk"); + } +} + fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { haystack .windows(needle.len()) .position(|window| window == needle) } +#[cfg(unix)] +fn assert_test_mode(path: &Path, expected: u32) { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::metadata(path) + .expect("read test mode") + .permissions() + .mode() + & 0o7777, + expected, + "mode for {}", + path.display() + ); +} + +#[cfg(not(unix))] +fn assert_test_mode(path: &Path, _expected: u32) { + assert!( + fs::metadata(path) + .expect("read test permissions") + .permissions() + .readonly(), + "{} must be read-only", + path.display() + ); +} + fn make_removable(path: &Path) { let Ok(metadata) = fs::symlink_metadata(path) else { return; diff --git a/crates/localityd/tests/replica_materializer.rs b/crates/localityd/tests/replica_materializer.rs index 832db767..75b53180 100644 --- a/crates/localityd/tests/replica_materializer.rs +++ b/crates/localityd/tests/replica_materializer.rs @@ -7,6 +7,7 @@ use std::sync::Arc; #[cfg(unix)] use std::sync::atomic::AtomicBool; use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; use locality_core::portable::{ ExportAttemptId, LogicalPath, ProjectionFileKind, ProjectionId, SessionId, SourceAction, @@ -242,6 +243,8 @@ enum V2ArchiveMutation { ReceiptBodyDigestMismatch, ReceiptGenerationMismatch, ReceiptCountMismatch, + BodyCorrupt, + UnauthorizedExtraPath, } fn v2_offer_and_archive(mutation: V2ArchiveMutation) -> (SealedExportOffer, Vec) { @@ -371,7 +374,39 @@ fn v2_offer_and_archive(mutation: V2ArchiveMutation) -> (SealedExportOffer, Vec< builder .append_pax_extensions(pax.iter().map(|(key, value)| (*key, value.as_bytes()))) .expect("append file PAX metadata"); - append_test_member(&mut builder, &TestMember::file("docs/readme.md", body)); + let delivered_body = if matches!(mutation, V2ArchiveMutation::BodyCorrupt) { + b"scope corrupted!\n".as_slice() + } else { + body.as_slice() + }; + append_test_member( + &mut builder, + &TestMember::file("docs/readme.md", delivered_body), + ); + if matches!(mutation, V2ArchiveMutation::UnauthorizedExtraPath) { + let extra_body = b"not authorized\n"; + let extra_sha256 = sha256_label(Sha256::digest(extra_body).into()); + let extra_projection_id = ProjectionId::new("projection-unauthorized"); + let extra_pax = [ + (PAX_SOURCE_CONNECTION_ID, source_connection_id.as_str()), + (PAX_PROJECTION_ID, extra_projection_id.as_str()), + (PAX_WINNING_SCOPE_ORDINAL, "0"), + (PAX_FILE_KIND, "markdown"), + (PAX_EFFECTIVE_ACTIONS, "[\"read\"]"), + (PAX_CONTENT_SHA256, extra_sha256.as_str()), + ]; + builder + .append_pax_extensions( + extra_pax + .iter() + .map(|(key, value)| (*key, value.as_bytes())), + ) + .expect("append unauthorized file PAX metadata"); + append_test_member( + &mut builder, + &TestMember::file("docs/unauthorized.md", extra_body), + ); + } let terminal_control = ExportTerminalControlV2 { writable_metadata, completion_receipt: receipt, @@ -541,6 +576,16 @@ fn scope_authorized_malformed_metadata_receipts_and_order_roll_back() { V2ArchiveMutation::ReceiptCountMismatch, "completion receipt does not match sealed export offer", ), + ( + "v2-corrupt-body", + V2ArchiveMutation::BodyCorrupt, + "a file body does not match its content digest", + ), + ( + "v2-unauthorized-extra-path", + V2ArchiveMutation::UnauthorizedExtraPath, + "canonical export inventory does not match sealed offer", + ), ] { let root = TestDirectory::new(label); let (offer, archive) = v2_offer_and_archive(mutation); @@ -556,6 +601,328 @@ fn scope_authorized_malformed_metadata_receipts_and_order_roll_back() { } } +const REAL_WORLD_DIRECTORY_COUNT: u64 = 614; +const REAL_WORLD_FILE_COUNT: u64 = 667; +const REAL_WORLD_CONTENT_BYTES: u64 = 47 * 1024 * 1024; + +#[derive(Clone)] +struct StressFile { + order_key: CanonicalFileOrderKey, + file_kind: ProjectionFileKind, + byte_length: usize, + fill: u8, +} + +fn stress_file_body(file: &StressFile) -> Vec { + if file.order_key.logical_path.as_str().ends_with("page.md") { + return b"# General Teamspace\n\nReal-world scope fixture.\n".to_vec(); + } + vec![file.fill; file.byte_length] +} + +fn real_world_scope_offer_and_archive() -> (SealedExportOffer, Vec) { + let source_connection_id = SourceConnectionId::new("source-notion-real-world"); + let source_generations = vec![OrderedSourceGeneration { + ordinal: 0, + source_connection_id: source_connection_id.clone(), + source_generation_id: SourceGenerationId::new("generation-real-world") + .expect("generation ID"), + }]; + + let mut directories = (0..600) + .map(|index| format!("root-{index:03}")) + .collect::>(); + let mut deep_path = "root-000".to_string(); + for depth in 0..14 { + deep_path.push_str(&format!("/d{depth:02}")); + directories.push(deep_path.clone()); + } + assert_eq!(directories.len() as u64, REAL_WORLD_DIRECTORY_COUNT); + directories.sort_by_key(|path| CanonicalDirectoryOrderKey { + depth: path.split('/').count() as u32, + logical_path: LogicalPath::new(path).expect("stress directory path"), + }); + + let markdown = b"# General Teamspace\n\nReal-world scope fixture.\n"; + let binary_count = REAL_WORLD_FILE_COUNT as usize - 2; + let binary_bytes = REAL_WORLD_CONTENT_BYTES as usize - markdown.len(); + let base_binary_bytes = binary_bytes / binary_count; + let extra_binary_bytes = binary_bytes % binary_count; + let mut files = Vec::with_capacity(REAL_WORLD_FILE_COUNT as usize); + files.push(StressFile { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: Some(LogicalPath::new("root-000").expect("zero-byte parent")), + logical_path: LogicalPath::new("root-000/empty.bin").expect("zero-byte path"), + projection_id: ProjectionId::new("projection-empty"), + }, + file_kind: ProjectionFileKind::Binary, + byte_length: 0, + fill: 0, + }); + files.push(StressFile { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: Some(LogicalPath::new(&deep_path).expect("deep parent")), + logical_path: LogicalPath::new(format!("{deep_path}/page.md")) + .expect("deep Markdown path"), + projection_id: ProjectionId::new("projection-page"), + }, + file_kind: ProjectionFileKind::Markdown, + byte_length: markdown.len(), + fill: 0, + }); + for index in 0..binary_count { + let parent = format!("root-{:03}", index % 100); + files.push(StressFile { + order_key: CanonicalFileOrderKey { + winning_scope_ordinal: 0, + parent_path: Some(LogicalPath::new(&parent).expect("blob parent")), + logical_path: LogicalPath::new(format!("{parent}/blob-{index:04}.bin")) + .expect("blob path"), + projection_id: ProjectionId::new(format!("projection-blob-{index:04}")), + }, + file_kind: ProjectionFileKind::Binary, + byte_length: base_binary_bytes + usize::from(index < extra_binary_bytes), + fill: (index % 251) as u8, + }); + } + files.sort_by(|left, right| left.order_key.cmp(&right.order_key)); + assert_eq!(files.len() as u64, REAL_WORLD_FILE_COUNT); + + let mut builder = Builder::new(Vec::new()); + let mut records = + Vec::with_capacity((REAL_WORLD_DIRECTORY_COUNT + REAL_WORLD_FILE_COUNT + 1) as usize); + for path in &directories { + let order_key = CanonicalDirectoryOrderKey { + depth: path.split('/').count() as u32, + logical_path: LogicalPath::new(path).expect("stress directory path"), + }; + records.push(CanonicalExportRecord::Directory { + order_key: order_key.clone(), + }); + append_test_member(&mut builder, &TestMember::directory(format!("{path}/"))); + } + + let mut body_digest = DeliveredBodyDigestV2::new(REAL_WORLD_FILE_COUNT); + let actions = BTreeSet::from([SourceAction::Read]); + let mut materialized_bytes = 0_u64; + for file in &files { + let body = stress_file_body(file); + assert_eq!(body.len(), file.byte_length); + materialized_bytes += body.len() as u64; + let content_sha256 = sha256_label(Sha256::digest(&body).into()); + body_digest + .update_file(&file.order_key.projection_id, &body) + .expect("update stress body digest"); + records.push(CanonicalExportRecord::File { + order_key: file.order_key.clone(), + source_connection_id: source_connection_id.clone(), + file_kind: file.file_kind.clone(), + effective_actions: actions.clone(), + content_sha256: content_sha256.clone(), + byte_length: body.len() as u64, + }); + let file_kind = match file.file_kind { + ProjectionFileKind::Markdown => "markdown", + ProjectionFileKind::Binary => "binary", + _ => unreachable!("stress fixture only uses Markdown and binary files"), + }; + let pax = [ + (PAX_SOURCE_CONNECTION_ID, source_connection_id.as_str()), + (PAX_PROJECTION_ID, file.order_key.projection_id.as_str()), + (PAX_WINNING_SCOPE_ORDINAL, "0"), + (PAX_FILE_KIND, file_kind), + (PAX_EFFECTIVE_ACTIONS, "[\"read\"]"), + (PAX_CONTENT_SHA256, content_sha256.as_str()), + ]; + builder + .append_pax_extensions(pax.iter().map(|(key, value)| (*key, value.as_bytes()))) + .expect("append stress file PAX metadata"); + append_test_member( + &mut builder, + &TestMember::file(file.order_key.logical_path.as_str(), body), + ); + } + assert_eq!(materialized_bytes, REAL_WORLD_CONTENT_BYTES); + records.push(CanonicalExportRecord::Control { + order_key: CanonicalControlOrderKey { ordinal: 0 }, + member_path: locality_protocol::RESERVED_EXPORT_METADATA_PATH.to_string(), + }); + + let writable_metadata = ScopeAuthorizedWritableExportMetadata { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: SessionId::new("session-real-world"), + export_attempt_id: ExportAttemptId::new("attempt-real-world").expect("attempt ID"), + source_generations: source_generations.clone(), + writable_entries: Vec::new(), + }; + let inventory_sha256 = + canonical_export_inventory_sha256(&records).expect("canonical stress inventory"); + let writable_metadata_sha256 = + canonical_writable_metadata_sha256(&writable_metadata).expect("writable metadata digest"); + let archive_entry_count = REAL_WORLD_DIRECTORY_COUNT + REAL_WORLD_FILE_COUNT + 1; + let offer = SealedExportOffer { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: writable_metadata.session_id.clone(), + export_attempt_id: writable_metadata.export_attempt_id.clone(), + source_generations: source_generations.clone(), + media_type: "application/x-tar".to_string(), + content_encoding: TarContentEncoding::Identity, + limits: ExportAttemptLimits { + max_files: REAL_WORLD_FILE_COUNT, + max_directories: REAL_WORLD_DIRECTORY_COUNT, + max_content_bytes: REAL_WORLD_CONTENT_BYTES, + }, + control_entry_count: 1, + file_count: REAL_WORLD_FILE_COUNT, + directory_count: REAL_WORLD_DIRECTORY_COUNT, + archive_entry_count, + selected_content_bytes: REAL_WORLD_CONTENT_BYTES, + inventory_sha256: inventory_sha256.clone(), + writable_metadata_sha256: writable_metadata_sha256.clone(), + sealed_at: "2026-07-23T20:00:00Z".to_string(), + expires_at: "2026-07-23T20:10:00Z".to_string(), + }; + offer + .validate_inventory(&records) + .expect("valid stress offer inventory"); + let terminal_control = ExportTerminalControlV2 { + writable_metadata, + completion_receipt: ExportCompletionReceipt { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: offer.session_id.clone(), + export_attempt_id: offer.export_attempt_id.clone(), + source_generations, + inventory_sha256, + writable_metadata_sha256, + delivered_control_entry_count: 1, + delivered_file_count: REAL_WORLD_FILE_COUNT, + delivered_directory_count: REAL_WORLD_DIRECTORY_COUNT, + delivered_archive_entry_count: archive_entry_count, + delivered_content_bytes: REAL_WORLD_CONTENT_BYTES, + delivered_body_sha256: body_digest.finish().expect("stress body digest"), + completed_at: "2026-07-23T20:00:03Z".to_string(), + }, + }; + append_test_member( + &mut builder, + &TestMember::file( + locality_protocol::RESERVED_EXPORT_METADATA_PATH, + serde_json::to_vec(&terminal_control).expect("serialize stress terminal control"), + ), + ); + builder.finish().expect("finish real-world v2 tar"); + ( + offer, + builder.into_inner().expect("collect real-world v2 tar"), + ) +} + +#[test] +fn scope_authorized_real_world_archive_streams_identity_and_zstd_atomically() { + for (label, encoding, chunk_size) in [ + ( + "v2-real-world-identity", + ReplicaArchiveEncoding::Identity, + 4093, + ), + ("v2-real-world-zstd", ReplicaArchiveEncoding::Zstd, 8191), + ] { + let root = TestDirectory::new(label); + let (mut offer, tar) = real_world_scope_offer_and_archive(); + let archive = match encoding { + ReplicaArchiveEncoding::Identity => tar, + ReplicaArchiveEncoding::Zstd => { + offer.content_encoding = TarContentEncoding::Zstd; + zstd::stream::encode_all(tar.as_slice(), 1).expect("compress stress v2 tar") + } + }; + let started = Instant::now(); + let summary = materialize_scope_authorized_replica_archive( + ReplicaArchive::new( + encoding, + ChunkedReader { + inner: Cursor::new(archive), + chunk_size, + }, + ), + &root.destination(), + ReplicaMaterializationLimits::default(), + &offer, + ) + .expect("materialize real-world scope archive"); + eprintln!( + "{label}: materialized {} files, {} directories, and {} bytes in {:?}", + summary.files, + summary.directories, + summary.materialized_bytes, + started.elapsed() + ); + + assert_eq!(summary.entries, 1_282); + assert_eq!(summary.files, REAL_WORLD_FILE_COUNT); + assert_eq!(summary.directories, REAL_WORLD_DIRECTORY_COUNT); + assert_eq!(summary.materialized_bytes, REAL_WORLD_CONTENT_BYTES); + assert_eq!( + fs::read(root.destination().join("root-000/empty.bin")).expect("read empty file"), + b"" + ); + assert_eq!( + fs::read( + root.destination().join( + "root-000/d00/d01/d02/d03/d04/d05/d06/d07/d08/d09/d10/d11/d12/d13/page.md" + ) + ) + .expect("read deep Markdown file"), + b"# General Teamspace\n\nReal-world scope fixture.\n" + ); + assert_eq!( + fs::metadata(root.destination().join("root-099/blob-0099.bin")) + .expect("stat repeated-parent binary") + .len(), + 74_110 + ); + assert!(!root.destination().join(".loc").exists()); + assert_modes(&root.destination(), 0o555); + assert_modes(&root.destination().join("root-000"), 0o555); + assert_modes(&root.destination().join("root-000/empty.bin"), 0o444); + } +} + +#[test] +fn scope_authorized_repeat_initialization_preserves_first_publication() { + let root = TestDirectory::new("v2-repeat-destination"); + let (offer, archive) = v2_offer_and_archive(V2ArchiveMutation::None); + materialize_v2( + archive.clone(), + ReplicaArchiveEncoding::Identity, + &offer, + &root.destination(), + ) + .expect("first initialization publishes"); + + let error = materialize_v2( + archive, + ReplicaArchiveEncoding::Identity, + &offer, + &root.destination(), + ) + .expect_err("repeat initialization must not replace an existing destination"); + assert_eq!( + error, + format!( + "replica destination already exists: {}", + root.destination().display() + ) + ); + assert_eq!( + fs::read(root.destination().join("docs/readme.md")).expect("read first publication"), + b"scope authorized\n" + ); + assert!(!root.destination().join(".loc").exists()); +} + #[test] fn scope_authorized_truncation_rolls_back() { let root = TestDirectory::new("v2-truncated"); From 0725adbebfdfb270dd09002ac440506da499a30f Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 18:52:53 -0700 Subject: [PATCH 12/14] Retry interrupted sandbox exports once --- crates/loc-cli/src/sandbox.rs | 219 ++++++++++++++++++++++++-------- crates/loc-cli/tests/sandbox.rs | 123 +++++++++++++++++- 2 files changed, 289 insertions(+), 53 deletions(-) diff --git a/crates/loc-cli/src/sandbox.rs b/crates/loc-cli/src/sandbox.rs index af34fe5d..4ca5524c 100644 --- a/crates/loc-cli/src/sandbox.rs +++ b/crates/loc-cli/src/sandbox.rs @@ -22,8 +22,9 @@ use locality_protocol::{ }; use localityd::remote_truth::{ReplicaArchive, ReplicaArchiveEncoding}; use localityd::replica_materializer::{ - ExpectedReplicaMaterializationReceipt, ReplicaMaterializationLimits, - ReplicaMaterializationSummary, materialize_replica_archive_with_expected_receipt, + ExpectedReplicaMaterializationReceipt, ReplicaMaterializationError, + ReplicaMaterializationLimits, ReplicaMaterializationSummary, + materialize_replica_archive_with_expected_receipt, materialize_scope_authorized_replica_archive, }; use reqwest::StatusCode; @@ -44,6 +45,7 @@ const HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(30 * 60); const HTTP_READ_TIMEOUT: Duration = Duration::from_secs(60); const BOOTSTRAP_EXCHANGE_ATTEMPTS: usize = 2; const EXPORT_ATTEMPT_CREATION_ATTEMPTS: usize = 2; +const EXPORT_ATTEMPT_STREAM_ATTEMPTS: usize = 2; const BOOTSTRAP_IDEMPOTENCY_DOMAIN: &[u8] = b"locality.session-exchange-idempotency.v1\0"; const IDEMPOTENCY_KEY_HEADER: &str = "Idempotency-Key"; const EXPORT_READ_AHEAD_CHUNK_BYTES: usize = 64 * 1024; @@ -444,7 +446,7 @@ fn run_sandbox_init_internal( let status = client.session_status(&capability)?; mark_profile(&mut profile, PROFILE_SESSION_STATUS); let session = validate_status(&capability, &status)?; - let (encoding, response, limits, validation) = match session { + let (encoding, summary) = match session { ValidatedSandboxSession::Legacy { offer, expected_receipt, @@ -452,32 +454,103 @@ fn run_sandbox_init_internal( validate_encoding_preference(offer, content_encoding)?; let limits = limits_for_offer(offer)?; let (encoding, response) = client.open_export(&capability, offer, content_encoding)?; - ( - encoding, + mark_profile(&mut profile, PROFILE_EXPORT_OPEN_HEADERS); + let summary = materialize_export_response( response, + encoding, + &root, limits, - ExportValidation::Legacy(expected_receipt), + &ExportValidation::Legacy(expected_receipt), + profile.as_deref_mut(), ) + .map_err(|failure| failure.error)?; + (encoding, summary) } ValidatedSandboxSession::ScopeAuthorized => { let request = export_attempt_request(&capability, content_encoding)?; let offer = client.create_export_attempt(&capability, &request)?; validate_scope_offer(&capability, &request, &offer)?; let limits = limits_for_scope_offer(&offer)?; - let (encoding, response) = client.open_export_attempt(&capability, &offer)?; - ( - encoding, - response, - limits, - ExportValidation::ScopeAuthorized(offer), - ) + let validation = ExportValidation::ScopeAuthorized(offer); + let mut last_retryable_error = None; + let mut completed = None; + for attempt in 0..EXPORT_ATTEMPT_STREAM_ATTEMPTS { + let (encoding, response) = + match client.open_export_attempt(&capability, validation.scope_offer()) { + Ok(opened) => opened, + Err(failure) + if failure.retryable + && has_retry_remaining(attempt, EXPORT_ATTEMPT_STREAM_ATTEMPTS) => + { + last_retryable_error = Some(failure.error); + continue; + } + Err(failure) => return Err(failure.error), + }; + mark_profile(&mut profile, PROFILE_EXPORT_OPEN_HEADERS); + match materialize_export_response( + response, + encoding, + &root, + limits, + &validation, + profile.as_deref_mut(), + ) { + Ok(summary) => { + completed = Some((encoding, summary)); + break; + } + Err(failure) + if failure.retryable + && has_retry_remaining(attempt, EXPORT_ATTEMPT_STREAM_ATTEMPTS) => + { + last_retryable_error = Some(failure.error); + } + Err(failure) => return Err(failure.error), + } + } + completed.ok_or_else(|| { + last_retryable_error.unwrap_or_else(|| { + SandboxInitError::Materialization( + "sandbox export retry ended without a result".to_string(), + ) + }) + })? } }; - mark_profile(&mut profile, PROFILE_EXPORT_OPEN_HEADERS); + + Ok(report(&root, &capability, encoding, summary)) +} + +struct ExportStreamFailure { + error: SandboxInitError, + retryable: bool, +} + +impl ExportStreamFailure { + fn fatal(error: SandboxInitError) -> Self { + Self { + error, + retryable: false, + } + } +} + +fn materialize_export_response( + response: Response, + encoding: ReplicaArchiveEncoding, + root: &Path, + limits: ReplicaMaterializationLimits, + validation: &ExportValidation, + mut profile: Option<&mut SandboxInitProfile>, +) -> Result { let (body, mut producer) = - spawn_export_read_ahead(response).map_err(|error| SandboxInitError::Http { - operation: "session export read-ahead setup", - detail: error.to_string(), + spawn_export_read_ahead(response).map_err(|error| ExportStreamFailure { + error: SandboxInitError::Http { + operation: "session export read-ahead setup", + detail: error.to_string(), + }, + retryable: false, })?; let profiled_body = ProfiledExportBody::new(body, profile.as_deref_mut()); let archive = ReplicaArchive::new(encoding, profiled_body); @@ -485,21 +558,31 @@ fn run_sandbox_init_internal( ExportValidation::Legacy(expected_receipt) => { materialize_replica_archive_with_expected_receipt( archive, - &root, + root, limits, - expected_receipt, + *expected_receipt, ) } ExportValidation::ScopeAuthorized(offer) => { - materialize_scope_authorized_replica_archive(archive, &root, limits, &offer) + materialize_scope_authorized_replica_archive(archive, root, limits, offer) } - } - .map_err(|error| SandboxInitError::Materialization(error.to_string())); + }; let producer_outcome = producer.join(); - mark_profile(&mut profile, PROFILE_STREAM_DECODE_MATERIALIZE); - - let summary = match materialization { - Err(error) => return Err(error), + if let Some(profile) = profile { + profile.mark(PROFILE_STREAM_DECODE_MATERIALIZE); + } + + match materialization { + Err(error) => { + let retryable = matches!( + producer_outcome, + Ok(ReadAheadProducerOutcome::ErrorDelivered) + ) && is_retryable_truncated_materialization(&error); + Err(ExportStreamFailure { + error: SandboxInitError::Materialization(error.to_string()), + retryable, + }) + } Ok(summary) => { match producer_outcome { Ok(ReadAheadProducerOutcome::CleanEof) => {} @@ -507,21 +590,39 @@ fn run_sandbox_init_internal( ReadAheadProducerOutcome::ConsumerClosed | ReadAheadProducerOutcome::ErrorDelivered, ) => { - return Err(SandboxInitError::Materialization( - "sandbox export transport ended without a clean EOF".to_string(), - )); + return Err(ExportStreamFailure { + error: SandboxInitError::Materialization( + "sandbox export transport ended without a clean EOF".to_string(), + ), + retryable: false, + }); } Err(()) => { - return Err(SandboxInitError::Materialization( - "sandbox export read-ahead worker panicked".to_string(), - )); + return Err(ExportStreamFailure { + error: SandboxInitError::Materialization( + "sandbox export read-ahead worker panicked".to_string(), + ), + retryable: false, + }); } } - summary + Ok(summary) } - }; + } +} - Ok(report(&root, &capability, encoding, summary)) +fn is_retryable_truncated_materialization(error: &ReplicaMaterializationError) -> bool { + match error { + ReplicaMaterializationError::Decode(message) + | ReplicaMaterializationError::MalformedTar(message) => { + message.contains("sandbox export transport read failed") + } + ReplicaMaterializationError::MissingTarEndMarker => true, + ReplicaMaterializationError::Write { source, .. } => source + .to_string() + .contains("sandbox export transport read failed"), + _ => false, + } } fn mark_profile(profile: &mut Option<&mut SandboxInitProfile>, phase: &'static str) { @@ -770,6 +871,15 @@ enum ExportValidation { ScopeAuthorized(SealedExportOffer), } +impl ExportValidation { + fn scope_offer(&self) -> &SealedExportOffer { + match self { + Self::ScopeAuthorized(offer) => offer, + Self::Legacy(_) => unreachable!("legacy export validation has no scope offer"), + } + } +} + fn validate_status<'a>( capability: &SessionCapability, status: &'a SandboxSessionStatus, @@ -1275,7 +1385,7 @@ impl SandboxHttpClient { &self, capability: &SessionCapability, offer: &SealedExportOffer, - ) -> Result<(ReplicaArchiveEncoding, Response), SandboxInitError> { + ) -> Result<(ReplicaArchiveEncoding, Response), ExportStreamFailure> { let response = self .export_client .get(self.export_attempt_url( @@ -1292,22 +1402,31 @@ impl SandboxHttpClient { ) .bearer_auth(&capability.opaque_capability) .send() - .map_err(|error| SandboxInitError::Http { - operation: "export-attempt stream", - detail: error.without_url().to_string(), + .map_err(|error| { + let retryable = error.is_connect() || error.is_timeout() || error.is_body(); + ExportStreamFailure { + error: SandboxInitError::Http { + operation: "export-attempt stream", + detail: error.without_url().to_string(), + }, + retryable, + } })?; - ensure_success(&response, "export-attempt stream")?; - require_media_type(response.headers(), "export-attempt stream", TAR_MEDIA_TYPE)?; - let encoding = response_encoding(response.headers())?; + ensure_success(&response, "export-attempt stream").map_err(ExportStreamFailure::fatal)?; + require_media_type(response.headers(), "export-attempt stream", TAR_MEDIA_TYPE) + .map_err(ExportStreamFailure::fatal)?; + let encoding = response_encoding(response.headers()).map_err(ExportStreamFailure::fatal)?; if protocol_encoding(encoding) != offer.content_encoding { - return Err(SandboxInitError::UnsupportedExportEncoding(format!( - "{} (sealed {})", - encoding_name(encoding), - match offer.content_encoding { - TarContentEncoding::Identity => "identity", - TarContentEncoding::Zstd => "zstd", - } - ))); + return Err(ExportStreamFailure::fatal( + SandboxInitError::UnsupportedExportEncoding(format!( + "{} (sealed {})", + encoding_name(encoding), + match offer.content_encoding { + TarContentEncoding::Identity => "identity", + TarContentEncoding::Zstd => "zstd", + } + )), + )); } Ok((encoding, response)) } diff --git a/crates/loc-cli/tests/sandbox.rs b/crates/loc-cli/tests/sandbox.rs index 8c85f8da..1c08538f 100644 --- a/crates/loc-cli/tests/sandbox.rs +++ b/crates/loc-cli/tests/sandbox.rs @@ -425,6 +425,106 @@ fn scope_authorized_export_attempts_materialize_identity_and_zstd() { assert_ne!(idempotency_keys[0], idempotency_keys[1]); } +#[test] +fn scope_authorized_truncated_get_retries_the_same_attempt_once() { + let directory = TestDirectory::new("scope-truncated-retry"); + let capability = capability(); + let status = scope_ready_status(capability.session_id.clone()); + let (offer, tar) = scope_export_fixture(TarContentEncoding::Identity); + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ResponseFixture::json(&offer), + ResponseFixture::export("identity", tar.clone()) + .with_declared_content_length(tar.len() + 128), + ResponseFixture::export("identity", tar), + ]); + + let report = run_sandbox_init_with_encoding( + SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }, + SandboxBootstrapToken::new("scope-retry-bootstrap").expect("token"), + SandboxContentEncodingPreference::Identity, + ) + .expect("retry the exact sealed export attempt"); + + assert_eq!(report.files, 1); + assert_eq!( + fs::read(directory.root().join("readme.md")).expect("read published retry"), + b"scope v2\n" + ); + assert_no_staging_directories( + directory + .root() + .parent() + .expect("replica destination parent"), + ); + + let bootstrap = server.request(); + let status = server.request(); + let attempt = server.request(); + let first_get = server.request(); + let second_get = server.request(); + assert_eq!(bootstrap.method, "POST"); + assert_eq!(status.method, "GET"); + assert_eq!(attempt.method, "POST"); + assert_eq!(attempt.path, "/v1/sessions/session-7/export-attempts"); + assert_eq!(first_get.method, "GET"); + assert_eq!(second_get.method, "GET"); + assert_eq!(first_get.path, second_get.path); + assert_eq!( + first_get.path, + "/v1/sessions/session-7/export-attempts/attempt-scope/export" + ); +} + +#[test] +fn scope_authorized_two_truncated_gets_fail_without_stale_tree() { + let directory = TestDirectory::new("scope-truncated-twice"); + let capability = capability(); + let status = scope_ready_status(capability.session_id.clone()); + let (offer, tar) = scope_export_fixture(TarContentEncoding::Identity); + let declared_length = tar.len() + 128; + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ResponseFixture::json(&offer), + ResponseFixture::export("identity", tar.clone()) + .with_declared_content_length(declared_length), + ResponseFixture::export("identity", tar).with_declared_content_length(declared_length), + ]); + + let error = run_sandbox_init_with_encoding( + SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }, + SandboxBootstrapToken::new("scope-retry-bootstrap").expect("token"), + SandboxContentEncodingPreference::Identity, + ) + .expect_err("two truncated responses must exhaust the bounded retry"); + + assert_eq!(error.code(), "materialization_failed"); + assert!(!directory.root().exists()); + assert_no_staging_directories( + directory + .root() + .parent() + .expect("replica destination parent"), + ); + let _bootstrap = server.request(); + let _status = server.request(); + let attempt = server.request(); + let first_get = server.request(); + let second_get = server.request(); + assert_eq!(attempt.method, "POST"); + assert_eq!(first_get.method, "GET"); + assert_eq!(second_get.method, "GET"); + assert_eq!(first_get.path, second_get.path); +} + #[test] fn scope_authorized_real_world_export_streams_through_the_client() { let directory = TestDirectory::new("scope-real-world"); @@ -606,9 +706,16 @@ fn scope_authorized_extra_path_is_rejected_without_partial_publication() { assert_eq!(error.code(), "materialization_failed"); assert!(!error.to_string().contains("scope-unauthorized-bootstrap")); assert!(!directory.root().exists()); - for _ in 0..4 { - let _ = server.request(); - } + let _bootstrap = server.request(); + let _status = server.request(); + let attempt = server.request(); + let export = server.request(); + assert_eq!(attempt.method, "POST"); + assert_eq!(export.method, "GET"); + assert_eq!( + export.path, + "/v1/sessions/session-7/export-attempts/attempt-scope/export" + ); } #[test] @@ -2145,6 +2252,16 @@ fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { .position(|window| window == needle) } +fn assert_no_staging_directories(parent: &Path) { + let staging = fs::read_dir(parent) + .expect("read replica parent") + .filter_map(Result::ok) + .map(|entry| entry.file_name()) + .filter(|name| name.to_string_lossy().starts_with(".locality-stage-")) + .collect::>(); + assert!(staging.is_empty(), "stale staging directories: {staging:?}"); +} + #[cfg(unix)] fn assert_test_mode(path: &Path, expected: u32) { use std::os::unix::fs::PermissionsExt; From 4f549c32355223333d55ce548da596390b06f15a Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 19:33:52 -0700 Subject: [PATCH 13/14] Retry cleanly truncated scope exports once --- crates/loc-cli/src/sandbox.rs | 14 ++-- crates/loc-cli/tests/sandbox.rs | 120 ++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 4 deletions(-) diff --git a/crates/loc-cli/src/sandbox.rs b/crates/loc-cli/src/sandbox.rs index 4ca5524c..2f08518a 100644 --- a/crates/loc-cli/src/sandbox.rs +++ b/crates/loc-cli/src/sandbox.rs @@ -574,10 +574,16 @@ fn materialize_export_response( match materialization { Err(error) => { - let retryable = matches!( - producer_outcome, - Ok(ReadAheadProducerOutcome::ErrorDelivered) - ) && is_retryable_truncated_materialization(&error); + let retryable = matches!(validation, ExportValidation::ScopeAuthorized(_)) + && match producer_outcome { + Ok(ReadAheadProducerOutcome::ErrorDelivered) => { + is_retryable_truncated_materialization(&error) + } + Ok(ReadAheadProducerOutcome::CleanEof) => { + matches!(&error, ReplicaMaterializationError::MissingTarEndMarker) + } + Ok(ReadAheadProducerOutcome::ConsumerClosed) | Err(()) => false, + }; Err(ExportStreamFailure { error: SandboxInitError::Materialization(error.to_string()), retryable, diff --git a/crates/loc-cli/tests/sandbox.rs b/crates/loc-cli/tests/sandbox.rs index 1c08538f..2209dd0f 100644 --- a/crates/loc-cli/tests/sandbox.rs +++ b/crates/loc-cli/tests/sandbox.rs @@ -525,6 +525,110 @@ fn scope_authorized_two_truncated_gets_fail_without_stale_tree() { assert_eq!(first_get.path, second_get.path); } +#[test] +fn scope_authorized_clean_eof_missing_tar_end_marker_retries_the_same_attempt_once() { + let directory = TestDirectory::new("scope-clean-eof-retry"); + let capability = capability(); + let status = scope_ready_status(capability.session_id.clone()); + let (offer, tar) = scope_export_fixture(TarContentEncoding::Identity); + let truncated = without_second_tar_end_block(&tar); + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ResponseFixture::json(&offer), + ResponseFixture::export("identity", truncated), + ResponseFixture::export("identity", tar), + ]); + + let report = run_sandbox_init_with_encoding( + SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }, + SandboxBootstrapToken::new("scope-clean-eof-bootstrap").expect("token"), + SandboxContentEncodingPreference::Identity, + ) + .expect("retry the clean-EOF truncated sealed export attempt"); + + assert_eq!(report.files, 1); + assert_eq!( + fs::read(directory.root().join("readme.md")).expect("read published retry"), + b"scope v2\n" + ); + assert_no_staging_directories( + directory + .root() + .parent() + .expect("replica destination parent"), + ); + + let bootstrap = server.request(); + let status = server.request(); + let attempt = server.request(); + let first_get = server.request(); + let second_get = server.request(); + assert_eq!(bootstrap.method, "POST"); + assert_eq!(status.method, "GET"); + assert_eq!(attempt.method, "POST"); + assert_eq!(attempt.path, "/v1/sessions/session-7/export-attempts"); + assert_eq!(first_get.method, "GET"); + assert_eq!(second_get.method, "GET"); + assert_eq!(first_get.path, second_get.path); + assert_eq!( + first_get.path, + "/v1/sessions/session-7/export-attempts/attempt-scope/export" + ); +} + +#[test] +fn scope_authorized_two_clean_eof_missing_tar_end_markers_exhaust_the_bounded_retry() { + let directory = TestDirectory::new("scope-clean-eof-twice"); + let capability = capability(); + let status = scope_ready_status(capability.session_id.clone()); + let (offer, tar) = scope_export_fixture(TarContentEncoding::Identity); + let truncated = without_second_tar_end_block(&tar); + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ResponseFixture::json(&offer), + ResponseFixture::export("identity", truncated.clone()), + ResponseFixture::export("identity", truncated), + ]); + + let error = run_sandbox_init_with_encoding( + SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }, + SandboxBootstrapToken::new("scope-clean-eof-bootstrap").expect("token"), + SandboxContentEncodingPreference::Identity, + ) + .expect_err("two clean-EOF truncated responses must exhaust the bounded retry"); + + assert_eq!(error.code(), "materialization_failed"); + assert!(!directory.root().exists()); + assert_no_staging_directories( + directory + .root() + .parent() + .expect("replica destination parent"), + ); + let _bootstrap = server.request(); + let _status = server.request(); + let attempt = server.request(); + let first_get = server.request(); + let second_get = server.request(); + assert_eq!(attempt.method, "POST"); + assert_eq!(attempt.path, "/v1/sessions/session-7/export-attempts"); + assert_eq!(first_get.method, "GET"); + assert_eq!(second_get.method, "GET"); + assert_eq!(first_get.path, second_get.path); + assert_eq!( + first_get.path, + "/v1/sessions/session-7/export-attempts/attempt-scope/export" + ); +} + #[test] fn scope_authorized_real_world_export_streams_through_the_client() { let directory = TestDirectory::new("scope-real-world"); @@ -2134,6 +2238,22 @@ fn tar_file(path: &[u8], body: &[u8]) -> Vec { builder.into_inner().expect("collect tar fixture") } +fn without_second_tar_end_block(tar: &[u8]) -> Vec { + const TAR_BLOCK_BYTES: usize = 512; + + let last_nonzero = tar + .iter() + .rposition(|byte| *byte != 0) + .expect("tar fixture contains a non-zero member"); + let member_end = (last_nonzero + 1).div_ceil(TAR_BLOCK_BYTES) * TAR_BLOCK_BYTES; + let truncated_len = member_end + TAR_BLOCK_BYTES; + assert!( + tar.len() >= truncated_len + TAR_BLOCK_BYTES, + "tar fixture must contain both end-marker blocks" + ); + tar[..truncated_len].to_vec() +} + fn read_request(stream: &mut TcpStream) -> CapturedRequest { stream .set_read_timeout(Some(Duration::from_secs(5))) From 798626e550c4d68561487f445b21a12fde50cf23 Mon Sep 17 00:00:00 2001 From: misrasaurabh1 Date: Thu, 23 Jul 2026 20:21:13 -0700 Subject: [PATCH 14/14] Negotiate scope export attempt limits --- crates/loc-cli/src/sandbox.rs | 63 +++++-- crates/loc-cli/tests/sandbox.rs | 165 ++++++++++++++++-- .../fixtures/sandbox-session-status-v2.json | 39 +++++ crates/locality-protocol/src/lib.rs | 4 + crates/locality-protocol/tests/golden.rs | 37 +++- 5 files changed, 274 insertions(+), 34 deletions(-) create mode 100644 crates/locality-protocol/fixtures/sandbox-session-status-v2.json diff --git a/crates/loc-cli/src/sandbox.rs b/crates/loc-cli/src/sandbox.rs index 2f08518a..130aa99b 100644 --- a/crates/loc-cli/src/sandbox.rs +++ b/crates/loc-cli/src/sandbox.rs @@ -466,8 +466,11 @@ fn run_sandbox_init_internal( .map_err(|failure| failure.error)?; (encoding, summary) } - ValidatedSandboxSession::ScopeAuthorized => { - let request = export_attempt_request(&capability, content_encoding)?; + ValidatedSandboxSession::ScopeAuthorized { + export_attempt_limits, + } => { + let request = + export_attempt_request(&capability, content_encoding, export_attempt_limits)?; let offer = client.create_export_attempt(&capability, &request)?; validate_scope_offer(&capability, &request, &offer)?; let limits = limits_for_scope_offer(&offer)?; @@ -869,7 +872,9 @@ enum ValidatedSandboxSession<'a> { offer: &'a TarExportOffer, expected_receipt: ExpectedReplicaMaterializationReceipt, }, - ScopeAuthorized, + ScopeAuthorized { + export_attempt_limits: &'a ExportAttemptLimits, + }, } enum ExportValidation { @@ -918,7 +923,17 @@ fn validate_status<'a>( "scope-authorized status contains a legacy export offer", )); } - return Ok(ValidatedSandboxSession::ScopeAuthorized); + let export_attempt_limits = + status + .export_attempt_limits + .as_ref() + .ok_or(SandboxInitError::InvalidReadySession( + "scope-authorized export-attempt limits are missing", + ))?; + validate_negotiated_export_attempt_limits(export_attempt_limits)?; + return Ok(ValidatedSandboxSession::ScopeAuthorized { + export_attempt_limits, + }); } let offer = status .export_offer @@ -936,8 +951,8 @@ fn validate_status<'a>( fn export_attempt_request( capability: &SessionCapability, preference: SandboxContentEncodingPreference, + limits: &ExportAttemptLimits, ) -> Result { - let defaults = ReplicaMaterializationLimits::default(); let request = ExportAttemptRequest { versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, opaque_session_capability: capability.opaque_capability.clone(), @@ -947,11 +962,7 @@ fn export_attempt_request( | SandboxContentEncodingPreference::Zstd => TarContentEncoding::Zstd, SandboxContentEncodingPreference::Identity => TarContentEncoding::Identity, }, - limits: ExportAttemptLimits { - max_files: defaults.max_entries.saturating_sub(1), - max_directories: defaults.max_entries.saturating_sub(1), - max_content_bytes: defaults.max_disk_bytes, - }, + limits: limits.clone(), }; request.validate().map_err(|_| { SandboxInitError::InvalidExportOffer("client export-attempt request is invalid") @@ -959,6 +970,38 @@ fn export_attempt_request( Ok(request) } +fn validate_negotiated_export_attempt_limits( + limits: &ExportAttemptLimits, +) -> Result<(), SandboxInitError> { + limits.validate().map_err(|_| { + SandboxInitError::InvalidReadySession("scope-authorized export-attempt limits are invalid") + })?; + let defaults = ReplicaMaterializationLimits::default(); + let maximum_entries = defaults.max_entries.saturating_sub(1); + for (limit, offered, maximum) in [ + ("maximum file count", limits.max_files, maximum_entries), + ( + "maximum directory count", + limits.max_directories, + maximum_entries, + ), + ( + "maximum content bytes", + limits.max_content_bytes, + defaults.max_disk_bytes, + ), + ] { + if offered > maximum { + return Err(SandboxInitError::ExportLimit { + limit, + offered, + maximum, + }); + } + } + Ok(()) +} + fn random_export_idempotency_key() -> Result { let mut random = [0_u8; 32]; rustls::crypto::ring::default_provider() diff --git a/crates/loc-cli/tests/sandbox.rs b/crates/loc-cli/tests/sandbox.rs index 2209dd0f..88efd423 100644 --- a/crates/loc-cli/tests/sandbox.rs +++ b/crates/loc-cli/tests/sandbox.rs @@ -407,6 +407,7 @@ fn scope_authorized_export_attempts_materialize_identity_and_zstd() { request.validate().expect("valid export-attempt request"); assert_eq!(request.content_encoding, protocol_encoding); assert_eq!(request.opaque_session_capability, "capability-secret"); + assert_eq!(request.limits, scope_attempt_limits()); assert!(request.idempotency_key.starts_with("loc-export-v2-")); assert_eq!(request.idempotency_key.len(), 78); idempotency_keys.push(request.idempotency_key); @@ -480,6 +481,139 @@ fn scope_authorized_truncated_get_retries_the_same_attempt_once() { ); } +#[test] +fn scope_authorized_status_requires_valid_export_attempt_limits() { + let cases = [ + ("missing", None), + ( + "zero-files", + Some(ExportAttemptLimits { + max_files: 0, + ..scope_attempt_limits() + }), + ), + ( + "zero-directories", + Some(ExportAttemptLimits { + max_directories: 0, + ..scope_attempt_limits() + }), + ), + ( + "zero-content-bytes", + Some(ExportAttemptLimits { + max_content_bytes: 0, + ..scope_attempt_limits() + }), + ), + ]; + + for (label, export_attempt_limits) in cases { + let directory = TestDirectory::new(label); + let capability = capability(); + let mut status = scope_ready_status(capability.session_id.clone()); + status.export_attempt_limits = export_attempt_limits; + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ]); + + let error = run_sandbox_init( + SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }, + SandboxBootstrapToken::new("scope-limits-bootstrap").expect("token"), + ) + .expect_err("missing or zero scope limits must fail before export-attempt creation"); + + assert_eq!(error.code(), "backend_protocol_invalid", "case {label}"); + assert_eq!( + error.to_string(), + if label == "missing" { + "ready sandbox session is invalid: scope-authorized export-attempt limits are missing" + } else { + "ready sandbox session is invalid: scope-authorized export-attempt limits are invalid" + }, + "case {label}" + ); + assert!(!directory.root().exists(), "case {label}"); + let _ = server.request(); + let _ = server.request(); + server.assert_no_request(); + } +} + +#[test] +fn scope_authorized_status_rejects_attempt_limits_above_client_safety_ceiling() { + let maximum = ReplicaMaterializationLimits::default() + .max_entries + .saturating_sub(1); + let cases = [ + ( + "files", + ExportAttemptLimits { + max_files: maximum + 1, + ..scope_attempt_limits() + }, + format!( + "sandbox export maximum file count {} exceeds client maximum {maximum}", + maximum + 1 + ), + ), + ( + "directories", + ExportAttemptLimits { + max_directories: maximum + 1, + ..scope_attempt_limits() + }, + format!( + "sandbox export maximum directory count {} exceeds client maximum {maximum}", + maximum + 1 + ), + ), + ( + "content-bytes", + ExportAttemptLimits { + max_content_bytes: ReplicaMaterializationLimits::default().max_disk_bytes + 1, + ..scope_attempt_limits() + }, + format!( + "sandbox export maximum content bytes {} exceeds client maximum {}", + ReplicaMaterializationLimits::default().max_disk_bytes + 1, + ReplicaMaterializationLimits::default().max_disk_bytes + ), + ), + ]; + + for (label, export_attempt_limits, expected_message) in cases { + let directory = TestDirectory::new(label); + let capability = capability(); + let mut status = scope_ready_status(capability.session_id.clone()); + status.export_attempt_limits = Some(export_attempt_limits); + let server = MockServer::start(vec![ + ResponseFixture::json(&capability), + ResponseFixture::json(&status), + ]); + + let error = run_sandbox_init( + SandboxInitOptions { + api_url: server.api_url.clone(), + root: directory.root(), + }, + SandboxBootstrapToken::new("scope-limits-bootstrap").expect("token"), + ) + .expect_err("server limits above local safety ceilings must fail before export"); + + assert_eq!(error.code(), "export_limit_exceeded", "case {label}"); + assert_eq!(error.to_string(), expected_message, "case {label}"); + assert!(!directory.root().exists(), "case {label}"); + let _ = server.request(); + let _ = server.request(); + server.assert_no_request(); + } +} + #[test] fn scope_authorized_two_truncated_gets_fail_without_stale_tree() { let directory = TestDirectory::new("scope-truncated-twice"); @@ -1767,6 +1901,7 @@ fn ready_status( decoded_bytes: decoded_tar.len() as u64, decoded_tar_sha256: sha256_label(decoded_tar), }), + export_attempt_limits: None, error: None, updated_at: "2026-07-20T11:00:00Z".to_string(), } @@ -1779,12 +1914,21 @@ fn scope_ready_status(session_id: SessionId) -> SandboxSessionStatus { state: SandboxSessionState::Ready, freshness_requirement: freshness_requirement(), replicas: Vec::new(), + export_attempt_limits: Some(scope_attempt_limits()), export_offer: None, error: None, updated_at: "2026-07-23T20:00:00Z".to_string(), } } +fn scope_attempt_limits() -> ExportAttemptLimits { + ExportAttemptLimits { + max_files: 88_001, + max_directories: 77_002, + max_content_bytes: 987_654_321, + } +} + fn scope_export_fixture(encoding: TarContentEncoding) -> (SealedExportOffer, Vec) { scope_export_fixture_with_unauthorized_extra(encoding, false) } @@ -1821,15 +1965,7 @@ fn scope_export_fixture_with_unauthorized_extra( source_connection_id: source_connection_id.clone(), source_generation_id: SourceGenerationId::new("generation-scope").expect("generation ID"), }]; - let limits = ExportAttemptLimits { - max_files: ReplicaMaterializationLimits::default() - .max_entries - .saturating_sub(1), - max_directories: ReplicaMaterializationLimits::default() - .max_entries - .saturating_sub(1), - max_content_bytes: ReplicaMaterializationLimits::default().max_disk_bytes, - }; + let limits = scope_attempt_limits(); let inventory_sha256 = canonical_export_inventory_sha256(&records).expect("scope inventory"); let writable_metadata = ScopeAuthorizedWritableExportMetadata { versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, @@ -2092,15 +2228,7 @@ fn real_world_scope_export_fixture(encoding: TarContentEncoding) -> (SealedExpor source_generations: source_generations.clone(), media_type: "application/x-tar".to_string(), content_encoding: encoding, - limits: ExportAttemptLimits { - max_files: ReplicaMaterializationLimits::default() - .max_entries - .saturating_sub(1), - max_directories: ReplicaMaterializationLimits::default() - .max_entries - .saturating_sub(1), - max_content_bytes: ReplicaMaterializationLimits::default().max_disk_bytes, - }, + limits: scope_attempt_limits(), control_entry_count: 1, file_count: FILE_COUNT, directory_count: DIRECTORY_COUNT, @@ -2198,6 +2326,7 @@ fn non_ready_status() -> SandboxSessionStatus { state: SandboxSessionState::Bootstrapping, freshness_requirement: freshness_requirement(), replicas: Vec::new(), + export_attempt_limits: None, export_offer: None, error: Some(SessionProtocolError { code: SessionErrorCode::Bootstrapping, diff --git a/crates/locality-protocol/fixtures/sandbox-session-status-v2.json b/crates/locality-protocol/fixtures/sandbox-session-status-v2.json new file mode 100644 index 00000000..d900e240 --- /dev/null +++ b/crates/locality-protocol/fixtures/sandbox-session-status-v2.json @@ -0,0 +1,39 @@ +{ + "versions": { + "session": 2, + "replica": 2, + "export_metadata": 2, + "writable_session_store": 1, + "canonical": 1, + "path": 1, + "changeset": 1 + }, + "session_id": "session-7", + "state": "ready", + "freshness_requirement": { + "max_age_seconds": 300, + "on_stale": "wait_then_fail", + "wait_timeout_seconds": 30 + }, + "replicas": [ + { + "source_connection_id": "source-notion", + "state": "fresh", + "coverage_complete": true, + "provider_observed_through": "notion-repair:108", + "last_successful_sync_at": "2026-07-19T11:58:00Z", + "last_repair_at": "2026-07-19T11:55:00Z", + "pending_events": 0, + "backlog": 0, + "provider_cooldown_until": null + } + ], + "export_attempt_limits": { + "max_files": 10000, + "max_directories": 10000, + "max_content_bytes": 104857600 + }, + "export_offer": null, + "error": null, + "updated_at": "2026-07-23T20:00:00Z" +} diff --git a/crates/locality-protocol/src/lib.rs b/crates/locality-protocol/src/lib.rs index db7ce482..1208a9ec 100644 --- a/crates/locality-protocol/src/lib.rs +++ b/crates/locality-protocol/src/lib.rs @@ -1588,6 +1588,8 @@ pub struct SandboxSessionStatus { pub state: SandboxSessionState, pub freshness_requirement: FreshnessRequirement, pub replicas: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub export_attempt_limits: Option, pub export_offer: Option, pub error: Option, pub updated_at: String, @@ -2198,6 +2200,8 @@ pub const BOOTSTRAP_EXCHANGE_GOLDEN_JSON: &[u8] = pub const FRESHNESS_STATUS_GOLDEN_JSON: &[u8] = include_bytes!("../fixtures/freshness-status.json"); pub const SANDBOX_SESSION_STATUS_GOLDEN_JSON: &[u8] = include_bytes!("../fixtures/sandbox-session-status.json"); +pub const SANDBOX_SESSION_STATUS_V2_GOLDEN_JSON: &[u8] = + include_bytes!("../fixtures/sandbox-session-status-v2.json"); pub const SESSION_PROTOCOL_ERROR_GOLDEN_JSON: &[u8] = include_bytes!("../fixtures/session-protocol-error.json"); pub const TAR_EXPORT_OFFER_GOLDEN_JSON: &[u8] = include_bytes!("../fixtures/tar-export-offer.json"); diff --git a/crates/locality-protocol/tests/golden.rs b/crates/locality-protocol/tests/golden.rs index 4182d016..e7d69ba5 100644 --- a/crates/locality-protocol/tests/golden.rs +++ b/crates/locality-protocol/tests/golden.rs @@ -28,12 +28,13 @@ use locality_protocol::{ OrderedExportRow, OrderedSourceGeneration, PROJECTION_VERSION_GOLDEN_JSON, ProjectionVersionContract, ProviderSourceScopeSelector, READY_REPLICA_REVISION_GOLDEN_JSON, ReadyReplicaRevision, ReplicaFreshnessState, ReplicaFreshnessStatus, - SANDBOX_SESSION_STATUS_GOLDEN_JSON, SCOPE_AUTHORIZED_COMPONENT_VERSIONS, - SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, SEALED_EXPORT_OFFER_GOLDEN_JSON, - SESSION_PROTOCOL_ERROR_GOLDEN_JSON, SOURCE_VERSION_GOLDEN_JSON, SandboxSessionState, - SandboxSessionStatus, ScopeAuthorizedSessionQuery, ScopeAuthorizedWritableExportMetadata, - ScopeContractError, SealedExportOffer, SessionCapability, SessionErrorCode, - SessionProtocolError, SessionReplicaRevision, SourceVersionContract, StaleSessionBehavior, + SANDBOX_SESSION_STATUS_GOLDEN_JSON, SANDBOX_SESSION_STATUS_V2_GOLDEN_JSON, + SCOPE_AUTHORIZED_COMPONENT_VERSIONS, SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, + SEALED_EXPORT_OFFER_GOLDEN_JSON, SESSION_PROTOCOL_ERROR_GOLDEN_JSON, + SOURCE_VERSION_GOLDEN_JSON, SandboxSessionState, SandboxSessionStatus, + ScopeAuthorizedSessionQuery, ScopeAuthorizedWritableExportMetadata, ScopeContractError, + SealedExportOffer, SessionCapability, SessionErrorCode, SessionProtocolError, + SessionReplicaRevision, SourceVersionContract, StaleSessionBehavior, TAR_EXPORT_METADATA_GOLDEN_JSON, TAR_EXPORT_OFFER_GOLDEN_JSON, TarContentEncoding, TarExportMetadata, TarExportOffer, WRITABLE_EXPORT_METADATA_GOLDEN_JSON, WritableExportMetadata, WritableMetadataEntry, validate_canonical_export_records, @@ -74,6 +75,11 @@ fn authorized_session_query_is_exact_golden_bytes() { #[test] fn scope_authorized_export_contracts_are_exact_golden_bytes() { + assert_exact_round_trip( + SANDBOX_SESSION_STATUS_V2_GOLDEN_JSON, + &sandbox_session_status_v2(), + ); + let query = scope_authorized_query(); query.validate().expect("valid scope query"); assert_exact_round_trip(SCOPE_AUTHORIZED_SESSION_QUERY_GOLDEN_JSON, &query); @@ -564,12 +570,31 @@ fn sandbox_session_status() -> SandboxSessionStatus { wait_timeout_seconds: 30, }, replicas: vec![freshness_status()], + export_attempt_limits: None, export_offer: Some(tar_export_offer()), error: None, updated_at: "2026-07-19T12:00:00Z".to_string(), } } +fn sandbox_session_status_v2() -> SandboxSessionStatus { + SandboxSessionStatus { + versions: SCOPE_AUTHORIZED_COMPONENT_VERSIONS, + session_id: SessionId::new("session-7"), + state: SandboxSessionState::Ready, + freshness_requirement: FreshnessRequirement { + max_age_seconds: 300, + on_stale: StaleSessionBehavior::WaitThenFail, + wait_timeout_seconds: 30, + }, + replicas: vec![freshness_status()], + export_attempt_limits: Some(attempt_limits()), + export_offer: None, + error: None, + updated_at: "2026-07-23T20:00:00Z".to_string(), + } +} + fn needs_update_error() -> SessionProtocolError { SessionProtocolError { code: SessionErrorCode::NeedsUpdate,