From 306d91238709542af830cea06bc50ec3c671f9b9 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 11:59:49 +0800 Subject: [PATCH 01/32] feat: complete runtime protocol identity semantics --- Cargo.toml | 1 + README.md | 10 +- ...mplete-protocol-and-operation-semantics.md | 231 +++++++ docs/deep-test-plan.md | 618 ++++++++++++++++++ docs/implementation-plan.md | 162 +++++ src/conformance.rs | 2 +- src/contract/artifact.rs | 1 + src/contract/capabilities.rs | 23 +- src/contract/observation.rs | 61 +- src/contract/protocol.rs | 42 ++ src/contract/resource.rs | 15 +- src/contract/unit.rs | 2 +- src/driver.rs | 4 +- src/managed.rs | 90 ++- src/registry.rs | 20 +- src/state/file.rs | 8 +- tests/general_runtime.rs | 469 ++++++++++++- 17 files changed, 1700 insertions(+), 59 deletions(-) create mode 100644 docs/adr/0002-complete-protocol-and-operation-semantics.md create mode 100644 docs/deep-test-plan.md create mode 100644 docs/implementation-plan.md diff --git a/Cargo.toml b/Cargo.toml index 857e4e6..367e252 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "a3s-runtime" version = "0.2.0" edition = "2021" +rust-version = "1.85" description = "Provider-neutral execution contract and client for A3S runtimes" license = "MIT" repository = "https://github.com/A3S-Lab/Runtime" diff --git a/README.md b/README.md index 79564e4..6840d38 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,9 @@ stop replay, removal replay, and generation-aware absence for both lifecycle classes. Provider repositories remain responsible for crash injection, reconstruction, provider-specific security, and resource-leak tests. +See the [deep test plan](docs/deep-test-plan.md) for the full contract, +durability, real-provider, fault, performance, soak, and A3S OS release gates. + ## Architecture The contract and managed lifecycle are intentionally independent of product and @@ -213,8 +216,11 @@ managed durability and validation provider driver and external runtime ``` -See [ADR 0001](docs/adr/0001-general-runtime-contract.md) for the ownership, -identity, retry, and migration decisions behind the contract. +See [ADR 0001](docs/adr/0001-general-runtime-contract.md) for the general +ownership model, [ADR 0002](docs/adr/0002-complete-protocol-and-operation-semantics.md) +for the completed protocol and operation semantics, and the +[implementation plan](docs/implementation-plan.md) for the dependency-ordered +delivery tasks. ## Development diff --git a/docs/adr/0002-complete-protocol-and-operation-semantics.md b/docs/adr/0002-complete-protocol-and-operation-semantics.md new file mode 100644 index 0000000..52b7f0d --- /dev/null +++ b/docs/adr/0002-complete-protocol-and-operation-semantics.md @@ -0,0 +1,231 @@ +# ADR 0002: Complete Protocol and Operation Semantics + +- Status: Accepted +- Date: 2026-07-17 +- Decision owners: A3S Runtime maintainers + +## Context + +ADR 0001 established the general Task and Service model, immutable generations, +durable request identity, and provider-neutral ownership boundary. Source and +test-plan review found several places where the public API does not yet provide +enough information to prove those guarantees: + +- some top-level wire types have no schema identifier; +- capability provider identity uses a different grammar from `ProviderId` and + is not checked against the selected driver; +- a newer generation can overwrite durable identity before the old provider + resource has been reconciled; +- file locking protects individual writes but not the asynchronous provider + operation between reservation and completion; +- Task outputs have no matching capability and no reported byte size; +- exec has a request ID but no durable replay receipt; +- request receipts are embedded in one record with a hard 10,000-entry limit; +- deadlines are checked before reservation but do not bound lock waiting and + provider dispatch; +- log, exec, stop-after-loss, and operation-result state semantics are not + explicit enough to form complete test oracles. + +These are pre-1.0 breaking changes. They are resolved together so provider and +consumer integrations migrate once to one coherent contract. + +## Decision + +### 1. Every top-level wire type is explicitly versioned + +The following top-level values carry a `schema` field and reject unknown +fields: + +- capabilities; +- unit specifications and apply/action requests; +- observations, inspections, and removals; +- log queries and log chunks; +- exec requests and exec results; +- durable unit records and request receipts. + +Nested value objects remain versioned by their enclosing top-level schema. +Adding `size_bytes` to output artifacts changes observations to +`a3s.runtime.observation.v2`. Adding typed provider identity and output +capability changes capabilities to `a3s.runtime.capabilities.v3`. The durable +request-journal layout uses `a3s.runtime.unit-record.v2`. + +The Runtime core does not silently reinterpret v1 records. A caller that needs +old records owns an explicit archival decoder or migration before starting the +new client. + +### 2. Provider identity is one typed value + +`RuntimeCapabilities.provider_id` uses `ProviderId`, serialized as its existing +lowercase string representation. `RuntimeDriver` reports its stable +`ProviderId`, and `ManagedRuntimeClient` rejects capabilities whose ID differs +from the selected driver. + +`RuntimeProviderFactory` construction becomes asynchronous. The registry +validates the created client's capabilities and verifies that the reported ID +matches the registered factory before returning the client. Observations do +not repeat the provider ID: the validated selected client is the authority, +while `provider_resource_id` remains provider-scoped identity. + +### 3. One unit ID has one converged provider generation + +Applying generation N durably makes N the desired generation before provider +dispatch. A driver must reconcile provider resources so that, when apply +returns successfully, exactly one resource generation for that unit remains. +Older generations are retired idempotently by the driver. + +A transient overlap is permitted while the provider performs a handoff, but a +crash and exact retry must converge it to one resource. A caller that needs two +generations alive simultaneously, such as a rolling deployment, uses distinct +unit IDs and owns traffic switching above Runtime. + +This rule lets a provider discover stale resources from provider labels or +equivalent metadata even though the core has already persisted the new desired +record. + +### 4. A state store supplies a cross-process operation lease + +`RuntimeStateStore` exposes an owned, asynchronous, per-unit operation lease. +`ManagedRuntimeClient` holds it across reservation, provider dispatch, and +durable completion for apply, inspect, stop, remove, logs, and exec. + +The file store implements the lease with a separate owner-only advisory-lock +file. Record writes continue to use a shorter record lock, so completion does +not recursively acquire the operation lock. A process crash or cancelled +future releases the lease through operating-system file-handle cleanup. + +Same-unit operations are serialized across tasks and processes. Different unit +IDs remain parallel. Distributed stores must implement an equivalent fenced +lease; a no-op implementation is not conformant. + +### 5. Recovery identity changes only after confirmed loss + +An ambiguous acknowledgement leaves a pending receipt. Retrying it must +discover the same provider resource and may not substitute identity. + +If inspect proves that a previously observed provider resource is absent, the +core durably records `unknown`. A later same-generation apply may adopt one new +provider resource ID. Once that observation is completed, exact replay returns +the replacement without another provider create. + +### 6. Operation postconditions are explicit + +Apply returns a provider-backed observation that has advanced beyond +`accepted`: + +- a Task returns `succeeded` or `failed`; +- a Service returns `running`, `stopped`, `failed`, or `unknown`; +- `preparing`, `starting`, and `stopping` are inspectable intermediate states, + not successful apply results. + +A Service may be `running` but unhealthy. That is a truthful result and does +not satisfy `converges`. + +Stop returns `stopped`, an already terminal observation, or durable `unknown` +when the provider resource is confirmed lost. It may not report a still-active +state as a successful stop result. Stop never recreates an unknown resource. + +The shared conformance successful fixtures must converge; negative fixtures +also verify failed and unhealthy results. + +### 7. Task outputs are capability-gated and exact + +`RuntimeFeature::OutputArtifacts` advertises Task output collection. A Task +with nonempty `outputs` is rejected before reservation unless the feature is +present. + +`RuntimeOutputArtifact` includes `size_bytes`. A succeeded Task that requested +outputs must report exactly the requested names. Every artifact media type must +match its output specification, every artifact is digest-bound, and +`size_bytes` must not exceed `max_bytes`. A provider without output collection, +including the current Docker driver, does not advertise the feature. + +`IsolationLevel::Confidential` additionally requires the attestation feature. +Usage remains optional observation data unless a future specification requests +it explicitly. + +### 8. Logs and exec have different state policies + +Logs are readable for the current, non-removed generation in any lifecycle +state, including terminal and unknown. This permits postmortem logs. The +provider may return `NotFound` when retained provider logs no longer exist. +Removal closes the Runtime log surface for that unit generation. + +Exec requires the current, non-removed generation to be `running`. It is a +mutating request and therefore participates in durable request replay: + +- reserve an exec receipt before dispatch; +- an exact completed retry returns the stored result without re-execution; +- an ambiguous failure leaves the receipt pending; +- retrying a pending exec uses the same request ID, and an Exec-capable driver + must deduplicate or reattach that request; +- conflicting reuse of the request ID fails before provider dispatch. + +Exec results are stored in the request journal rather than embedded in the +unit record because bounded output may still be large. + +### 9. Request receipts use a durable per-request journal + +The v2 file layout separates the active unit record from receipts: + +```text +state root/ +├── locks/ +├── operations/ +└── units/ + └── / + ├── record.json + └── requests/ + └── .json +``` + +Pending and completed receipts are atomically written and owner-only. Exact +replay is guaranteed until an explicitly removed unit is purged through a +future administrative retention operation. Normal lifecycle methods never +silently discard receipts, and there is no 10,000-operation availability +cliff. + +The public unit record does not need to load every historical result to perform +an operation. Reservations return the one relevant receipt. Tests can enumerate +the journal through a test-only or administrative inspection surface. + +### 10. Deadlines cover queueing and provider work + +Managed operations validate the deadline before capability work, again after +acquiring the operation lease, and use the remaining duration to bound the +provider future. A timeout returns `DeadlineExceeded` and leaves a dispatched +mutating request pending because provider acknowledgement is ambiguous. + +Drivers may enforce a shorter provider-specific timeout. They must never extend +the caller deadline. Exec uses the smaller of its relative `timeout_ms` and an +optional absolute request deadline. + +Logs and inspect remain read operations without a request deadline in this +version; provider adapters must still have a configured transport timeout. + +## Conformance consequences + +The shared conformance suite is split into Base, Recovery, Networking, Mounts, +Health, Resources, Logs, Exec, Security, Outputs, and Evidence profiles. + +- Base and Recovery are mandatory for every production provider. +- An advertised optional capability activates its corresponding profile. +- A provider job fails if a required fixture or provider prerequisite is + absent; it does not silently pass by returning early. +- Generation advancement, operation cancellation, and all mutating crash + windows include provider inventory checks. +- Successful cleanup must return provider and state inventory to the declared + baseline. + +## Consequences + +- The public protocol and durable state schema break once before 1.0. +- Provider adapters must add typed identity, generation reconciliation, + operation timeout handling, and idempotent exec when advertised. +- State-store implementations gain a cross-process lease and separate receipt + journal. +- Exact replay no longer has an arbitrary embedded-record limit. +- Docker truthfully rejects requested outputs and exec until those capabilities + are implemented. +- A3S Box can implement `IsolationLevel::Sandbox` behind the same contract + without adding Box-specific fields. +- Tests can now derive a deterministic oracle for every public operation. diff --git a/docs/deep-test-plan.md b/docs/deep-test-plan.md new file mode 100644 index 0000000..26a0736 --- /dev/null +++ b/docs/deep-test-plan.md @@ -0,0 +1,618 @@ +# A3S Runtime Deep Test Plan + +## 1. Purpose + +This document defines the verification architecture for `A3S-Lab/Runtime`. +The target is not merely a green crate test suite. The target is evidence that +the provider-neutral contract, durable lifecycle coordinator, state store, real +providers, and consuming control paths preserve the same semantics through +retries, crashes, contention, scale, and provider loss. + +All build and runtime execution described here must run on Linux CI runners or +an A3S OS test runner. Developer laptops are not part of the execution matrix. +Servers must obtain source through Git at an exact commit; source archives must +not be copied from a workstation. + +## 2. Scope + +The plan covers five boundaries: + +1. The public contract in `src/contract/`. +2. `ManagedRuntimeClient`, `FileRuntimeStateStore`, provider registration, and + the exported conformance suite. +3. Every concrete `RuntimeDriver` against its real provider. +4. The A3S Cloud projection, node command journal, reconciliation, and + observation path that consume this crate. +5. Performance, fault recovery, security, resource cleanup, and production + canary evidence. + +The A3S OS product runtime APIs that happen to use the word "runtime" are not +automatically in scope. They enter this plan only when they construct or carry +the Rust `a3s-runtime` protocol. + +## 3. Evidence-Based Baseline + +Source inspection on 2026-07-17 established this baseline. It must be refreshed +at the start of an implementation or release campaign. + +| Area | Current evidence | Gap | +| --- | --- | --- | +| Runtime repository CI | No repository workflow is present | A green commit has no independent build or test evidence | +| Core integration tests | `tests/general_runtime.rs` contains 16 lifecycle, conflict, recovery, state, registry, and conformance scenarios | Most boundary combinations, crash points, cross-process races, and capacity limits are untested | +| Shared conformance | One happy-path Task and Service flow with replay checks | Optional features and negative/fault profiles are not covered | +| Real provider | A3S Cloud contains `DockerRuntimeDriver` | No other concrete Runtime driver is implemented | +| Real Docker tests | Three tests cover common conformance, create-before-state-update recovery, and external provider loss | They return success without running unless `A3S_CLOUD_TEST_DOCKER=1` | +| A3S Box | Documented as a future provider | No `RuntimeDriver` exists, so Box conformance cannot currently be claimed | +| Non-functional testing | No benchmark, fuzz, mutation, multi-process crash, or soak harness is present | No regression or durability evidence exists | + +Existing tests are useful regression assets, but their presence is not proof +that they passed for a given commit. Every report must bind results to an exact +Runtime commit, provider commit, provider build, fixture digest, and host. + +## 4. Required Invariants + +These invariants are the basis of every test oracle. + +### 4.1 Protocol and identity + +- Every wire record accepts only its declared schema and fields. +- `(unit_id, generation, canonical_spec_digest)` is immutable. +- A request ID identifies exactly one request kind and digest. +- Exact request replay returns the same logical result. +- Stale generations and conflicting content fail before provider mutation. +- A mutable artifact tag never replaces the declared digest. +- Capability rejection occurs before state reservation and provider dispatch. +- Provider observations, exec results, removals, evidence, and attestations + cannot substitute caller-owned identity. + +### 4.2 Lifecycle and recovery + +- Task convergence means `succeeded`; Service convergence means `running` and, + when configured, `healthy`. +- Terminal observations are immutable. +- Explicit removal creates a durable generation-aware tombstone. +- A missing previously observed provider resource becomes `unknown`, not + `not_found` or success. +- An ambiguous acknowledgement retries the same provider identity and must not + create a duplicate resource. +- After confirmed provider loss has been persisted as `unknown`, a same- + generation apply may adopt one replacement provider identity. Subsequent + replay must return that replacement without another create. +- A deadline at or before the current clock prevents reservation and dispatch. +- Provider operation timeouts remain bounded independently of the pre-dispatch + request deadline check. + +### 4.3 Durability and concurrency + +- A completed state write survives process restart and host reboot. +- A failed or interrupted write never destroys the last valid record. +- A pending receipt survives an ambiguous provider result. +- Concurrent operations preserve every accepted receipt and never create an + untracked provider resource. +- Same-unit serialization and different-unit parallelism have explicit, + measured behavior. +- State paths, files, locks, and temporary writes never follow a symbolic-link + boundary and retain owner-only permissions. +- Corrupt, truncated, mismatched, or future-schema state fails closed before + provider work. + +### 4.4 Provider truthfulness + +- Every advertised capability has a real passing test. +- Every unadvertised optional feature is rejected before provider dispatch. +- Resource limits are verified by provider inspection and observable behavior, + not only by request construction. +- A provider restart, agent restart, or external resource deletion converges to + one durable outcome with zero duplicate resources. +- A successful suite leaves no provider resources, state roots, ports, mounts, + processes, or test volumes outside its declared retention policy. + +## 5. Contract Decisions Required Before P0 Closes + +Tests must not encode accidental behavior. The maintainers must resolve and +record the following decisions in an ADR or contract documentation before the +corresponding release gate is enabled. + +| Decision | Current risk | Required test oracle | +| --- | --- | --- | +| Wire schema boundary | Top-level log, exec, and inspection types do not all carry schema identifiers although the README says all wire records do | Define which types cross a versioned boundary; require a schema on each top-level wire record or narrow the compatibility claim | +| Provider identity binding | Capabilities carry a string validated differently from `ProviderId`, and the managed client does not bind it to a selected factory | Use one grammar and prove reported, registered, and observed provider identities cannot disagree | +| Generation handoff | Reserving a newer generation replaces the stored observation before the old provider resource is necessarily stopped or removed | Define caller, managed client, or driver ownership of the prior resource; prove no orphan on success, error, or crash | +| Cross-operation concurrency | State locking does not span an asynchronous provider call | Define ordering for apply/apply, apply/stop, apply/remove, stop/remove, and generation races | +| Recovery identity | Recovery from `unknown` can differ from ambiguous-ack reattachment | Permit identity replacement only after durable `unknown`; all other identity changes fail | +| Operation postconditions | Apply rejects only `accepted`, while stop can accept any otherwise valid transition | Define whether calls return only converged results or may return transitional observations, then test each allowed result | +| Task output fulfillment | A Task may request outputs, but capabilities do not express output collection and the current Docker driver returns none | Make output support mandatory or capability-gated; prove every requested output is collected, bounded, and digest-bound before convergence | +| Logs by state | Current-generation logs may be useful after Task completion or Service stop | Define allowed states, removal behavior, cursor retention, and provider-loss behavior | +| Exec by state | Generation matching alone does not prove a runnable unit | Require `running`, or explicitly delegate a narrower rule to providers | +| Stop after loss | A provider may be absent while durable state is `unknown` | Define whether stop returns `unknown`, is idempotent success, or requires recovery | +| Receipt retention | Records reject more than 10,000 receipts but have no retention/compaction policy | Define bounded retention without breaking exact replay guarantees | +| Deadline semantics | Deadlines are checked only before dispatch | Define whether drivers receive remaining budget and how late results are persisted | + +## 6. Test Architecture + +The suite is divided into layers so fast deterministic checks run frequently +while destructive checks run only in isolated environments. + +| Layer | Purpose | Environment | Trigger | Maximum target duration | +| --- | --- | --- | --- | --- | +| L0 | Format, lint, docs, unit, golden protocol, property checks | Ephemeral Linux CI | Every pull request | 10 minutes | +| L1 | Managed lifecycle, file state, multi-process contention, deterministic fault driver | Ephemeral Linux CI | Every pull request | 20 minutes | +| L2 | Real provider capability and conformance profiles | Dedicated Linux provider runner | Merge and nightly | 30 minutes per provider | +| L3 | Crash, reboot, disk fault, scale, resource enforcement, and leak tests | Disposable A3S OS worker or VM | Nightly or weekly | 2 hours | +| L4 | Long soak and safe production canary | Dedicated soak worker; shared production only for non-destructive canary | Release candidate | 24 to 72 hours | + +No layer may silently convert an unavailable prerequisite into a passing test. +A skipped provider job must be reported as `SKIPPED`, and a release gate that +requires that provider must fail. + +The initial automation uses the crate's documented commands from the Runtime +repository root: + +```text +cargo fmt --all --check +cargo test --all-targets +cargo clippy --all-targets -- -D warnings +RUSTDOCFLAGS="-D warnings" cargo doc --no-deps +``` + +The real Docker gate runs from the A3S Cloud repository root: + +```text +A3S_CLOUD_TEST_DOCKER=1 cargo test -p a3s-cloud-node-agent \ + --test docker_conformance -- --nocapture --test-threads=1 +``` + +The Runtime crate must declare a minimum supported Rust version before release; +L0 tests that version and current stable on the production Linux architectures. + +## 7. Functional Test Matrix + +Each implemented case receives a stable ID in the test name and evidence +manifest. Boundary tests use `minimum-1`, `minimum`, `maximum`, and `maximum+1` +where the type permits those values. + +### 7.1 Serialization, validation, and digests + +| IDs | Required coverage | +| --- | --- | +| `CT-SCHEMA-*` | Correct, missing, old, future, malformed, and unknown-field behavior for every public record | +| `CT-ID-*` | Empty, 1 byte, maximum length, overlength, control characters, Unicode byte length, path-like characters, and provider ID grammar | +| `CT-DIGEST-*` | Digest algorithm, hexadecimal length, invalid bytes, deterministic output across processes and architectures, and one-field mutation sensitivity | +| `CT-JSON-*` | Golden JSON decode/encode, enum tags, field names, missing required fields, duplicate JSON keys, and map-order independence | +| `CT-URI-*` | Artifact URI schemes, authority/path boundaries, digest binding, credentials, query/fragment, and mutable tags in each provider adapter | +| `CT-PATH-*` | Absolute paths, repeated separators, dot segments, `..`, NUL, CR/LF, maximum length, and mount/secret/output collision policy | + +Golden fixtures are versioned test data. Updating one requires an explicit +schema review; snapshot acceptance is never automatic. + +### 7.2 Unit specifications + +| IDs | Required coverage | +| --- | --- | +| `SPEC-PROCESS-*` | Entrypoint fallback, command/argument counts and sizes, working directory, environment names/values, and deterministic map order | +| `SPEC-MOUNT-*` | Artifact, volume, and tmpfs sources; read-only behavior; zero/overflow size; duplicate names and targets; cross-kind collisions | +| `SPEC-SECRET-*` | Opaque references, environment/file targets, file modes, duplicate names/targets, collision with process environment and mounts, and non-disclosure | +| `SPEC-NET-*` | None/outbound/service modes, TCP/UDP, zero port, duplicate name/socket, 64-port boundary, and service-only publication | +| `SPEC-HEALTH-*` | HTTP/TCP/command probes, named-port binding, status ranges, path validation, timing relationships, start period, and thresholds | +| `SPEC-RESOURCE-*` | CPU, memory, PIDs, optional ephemeral storage, Task timeout, zero values, numeric maxima, and provider conversion overflow | +| `SPEC-ISOLATION-*` | Process, container, sandbox, and confidential requirements; capability rejection; provider mapping and evidence | +| `SPEC-RESTART-*` | Never, always, and on-failure policies; retry boundaries; Task/Service restrictions; provider enforcement | +| `SPEC-CLASS-*` | Every Task/Service restriction for health, timeout, restart policy, and outputs | +| `SPEC-OUTPUT-*` | Names, absolute paths, media types, size limits, duplicates, and succeeded-Task-only observations | +| `SPEC-SEMANTICS-*` | Optional profile digest validation and evidence binding | + +Ephemeral storage is conditional: a provider needs +`ResourceControl::EphemeralStorage` only when the specification requests a +quota. CPU, memory, and PIDs remain required for every unit. + +### 7.3 Capabilities and registry + +Test every required family, duplicate entry, malformed provider identity, +empty optional family, and complete missing-capability ordering. Generate a +spec that independently requires each enum value and optional feature. Verify +that the registry rejects duplicate providers, has no default, never falls +back, and surfaces factory construction failures unchanged. + +Capability claims are tested twice: + +1. Pure matching proves that the contract calculates requirements correctly. +2. Real-provider probes prove that each advertised claim works. + +### 7.4 Observations and transitions + +Generate and verify the full state-transition matrix for Task and Service. +Cover provider identity, build identity, monotonic observation time, start and +finish ordering, terminal timestamps, health, outputs, usage, evidence, +attestation, failure details, and convergence. + +The matrix must include: + +- every allowed transition; +- every forbidden transition; +- same-state refresh; +- all transitions to and from `unknown` allowed by the final contract; +- terminal equality replay and every attempted terminal mutation; +- provider identity substitution before and after confirmed loss; +- a Service attempting `succeeded` and a Task carrying health; +- duplicate output artifacts and evidence bound to another specification. + +### 7.5 Managed operations + +| IDs | Required scenarios | +| --- | --- | +| `LC-APPLY-*` | First apply; pending replay; completed replay; same generation/new request; generation conflict; stale and next generation; rejected capability; expired deadline | +| `LC-INSPECT-*` | Never seen; active refresh; terminal cache; removed tombstone; provider absent to durable `unknown`; malformed provider response | +| `LC-STOP-*` | Running, already stopped, terminal Task, absent, removed, unknown, exact replay, request conflict, unsupported feature, and deadline | +| `LC-REMOVE-*` | Running, stopped, Task terminal, already absent with new request, exact replay, identity substitution, and provider error | +| `LC-LOG-*` | Current/stale/future/removed generation, stdout/stderr filter, limits, strict sequence, cursor resume, invalid cursor, rotation gap, large chunk, and capability rejection | +| `LC-EXEC-*` | Allowed/forbidden states, timeout, exit range, stdout/stderr bounds, truncation, identity substitution, generation checks, replay policy, and capability rejection | + +Every mutation test records state immediately before reservation, after +reservation, after provider completion, and after durable completion. + +## 8. Durable State and Race Matrix + +### 8.1 Filesystem safety + +Run on a real Linux filesystem with hostile fixtures for the root, `locks`, +`units`, lock file, record file, and temporary-file boundary. Cover symbolic +links, hard links where relevant, non-regular files, permission changes, +different umasks, wrong ownership, read-only filesystem, `ENOSPC`, inode +exhaustion, truncated JSON, invalid UTF-8, unknown schema, key mismatch, and a +record exceeding the receipt limit. + +After every successful write, assert directory mode `0700`, file and lock mode +`0600`, valid JSON, matching storage key, and a directory sync. After every +injected failure, assert that either the prior record or the complete new record +is readable; a partially published record is never acceptable. + +### 8.2 Crash points + +Use subprocess tests and named failpoints around: + +1. lock acquisition; +2. initial reservation construction; +3. temporary-file creation; +4. partial and complete writes; +5. file sync; +6. permission tightening; +7. atomic publish; +8. directory sync; +9. provider create/start/stop/remove; +10. observation or removal completion. + +For each point, kill the process, create a new client from the same state root, +replay the same request, and compare provider inventory with durable state. +Cancel the caller future at each asynchronous boundary as a separate case; +dropping a future must not make a later state mutation or provider result +unrecoverable. + +### 8.3 Concurrency cases + +Run each race in thread, task, and independent-process variants where the +boundary permits it: + +- 2, 32, and 128 identical apply requests; +- same specification with distinct request IDs; +- conflicting content with one request ID; +- generations N and N+1 concurrently; +- apply versus inspect, stop, and remove; +- stop versus remove; +- observation refresh versus remove; +- 1,000 different units concurrently; +- 10,001 sequential receipts for one unit. + +Pass criteria are deterministic accepted/error classes, no lost completed +receipt, no invalid record, no deadlock, and exactly the provider resources +allowed by the generation-handoff decision. + +## 9. Provider Conformance Profiles + +Replace the single all-or-nothing conformance path with composable profiles. +The shared Runtime repository owns the oracles; provider repositories own +fixtures and destructive cleanup. + +| Profile | Mandatory evidence | +| --- | --- | +| Base | Valid capabilities, Task success/failure/timeout, Service start/inspect/stop/remove, exact replay, generation conflict, tombstone | +| Recovery | Create-before-ack crash, client restart, provider restart, external deletion to `unknown`, one same-generation replacement, duplicate-resource detection | +| Networking | Every advertised network mode and protocol, loopback publication, outbound denial/allowance, port collision behavior | +| Mounts | Every advertised mount kind, read-only enforcement, persistence, isolation, cleanup | +| Health | Every advertised probe kind, threshold transitions, timeout, start period, unhealthy exit | +| Resources | Every advertised control verified by provider configuration and workload behavior | +| Logs | Stream filtering, total order, cursor resume, same-timestamp records, limit, rotation gap, retention, large records | +| Exec | State policy, timeout, exit code, output bounds, truncation, identity and generation binding | +| Security | Digest pinning, label/metadata tamper, namespace separation, secret handling, least privilege, hostile input | +| Evidence | Usage, evidence claims, profile binding, attestation validity for each advertised optional feature | + +The harness must always run the Base and Recovery profiles. It discovers +optional profiles from capabilities and fails if an advertised capability has +no fixture or passing probe. + +## 10. Current Provider Matrix + +This table describes source claims, not release certification. + +| Capability | Docker driver in A3S Cloud | A3S Box driver | +| --- | --- | --- | +| Task and Service | Advertised | Not implemented | +| OCI/Docker manifest | Advertised with digest-pinned URI | Not implemented at this boundary | +| Isolation | `container` | Intended `sandbox`; no adapter evidence | +| Network | none, outbound, service | Not implemented at this boundary | +| Mounts | named volume, tmpfs | Not implemented at this boundary | +| Health | HTTP, TCP, command | Not implemented at this boundary | +| Resources | CPU, memory, PIDs, Task timeout | Not implemented at this boundary | +| Features | durable identity, stop, remove, logs | Not implemented at this boundary | +| Not advertised | exec, usage, attestation, secrets, artifact mounts, ephemeral quota | No claims are testable yet | + +An A3S Box row may become a release gate only after a real `RuntimeDriver` +exists and maps `IsolationLevel::Sandbox` without provider-specific fields in +the public contract. + +### 10.1 Docker-specific mandatory cases + +- Pull and reuse a digest-pinned image; reject tag-only, digest mismatch, + credentials in the URI, malformed registry responses, and interrupted pull. +- Validate all managed labels and reject label tampering, wrong node, + namespace collision, and multiple matching containers. +- Prove Task exit 0, nonzero exit, signal exit, timeout, and daemon error. +- Prove HTTP, TCP, and command health success and failure thresholds. +- Verify CPU, memory/swap, PIDs, network mode, loopback port binding, restart + policy, volume mode, and hardened tmpfs through Docker inspect and behavior. +- Exercise stdout/stderr ordering, nanosecond timestamp ties, cursor replay, + log rotation gap, filters, and one-MiB record rejection. +- Restart the node agent and Docker daemon at each recovery window. +- Count labeled containers and test volumes before and after every case; both + deltas must be zero after cleanup. + +The Docker job must set `A3S_CLOUD_TEST_DOCKER=1` explicitly and verify the +daemon before the test binary starts. A missing socket is a failed provider job, +not a passed test with early returns. + +## 11. Consumer End-to-End Matrix + +The first real consumer is A3S Cloud. Its release suite must prove: + +- an immutable `WorkloadRevision` projects to the expected Service spec and + digest; +- scheduler capability matching rejects an ineligible node before dispatch; +- the node command journal deduplicates exact commands and rejects conflicts; +- command lease expiry, redelivery, observation loss, and reordering converge; +- control-plane and node-agent restart preserve one provider resource; +- generation updates and rollback follow the generation-handoff decision; +- runtime observations remain bound to the selected node and deployment; +- ordered log cursors survive transport batching and reconnect; +- stop, cancel, deferred cleanup, and reconciliation leave no untracked unit. + +Bench or another caller must add a separate profile-projection suite before it +can claim Runtime compatibility. Product-specific scoring, privacy, and +scheduling remain outside the Runtime crate, but their projection digest and +generic resource/isolation requirements are in scope. + +## 12. Fault Injection Matrix + +| Fault | Expected result | +| --- | --- | +| Transport error before provider mutation | Pending receipt; exact retry dispatches safely | +| Provider create succeeds, acknowledgement is lost | Retry reattaches the same resource; count remains one | +| Provider resource is externally deleted | Inspect persists `unknown`; one reapply creates one replacement | +| Agent is killed at every state/provider boundary | Restart and replay converge without invalid state or orphan | +| Provider daemon restarts | Operations remain bounded and later reconcile without duplicates | +| Host reboots after file or provider sync | Durable state and provider inventory converge | +| Disk becomes full or read-only | Prior record remains valid; no provider work after failed reservation | +| State file is truncated or tampered | Fail closed; no provider mutation | +| Clock jumps backward/forward | Request deadline and observation monotonicity remain deterministic | +| Duplicate provider resources are injected | Inspect/apply fails closed and reports the invariant violation | +| Network is partitioned or latency exceeds timeout | Bounded error, durable pending state where acknowledgement is ambiguous | +| Caller future is cancelled or provider task panics | Durable state remains replayable and later work cannot publish an untracked result | + +Daemon restart, host reboot, disk faults, and broad network faults must never be +run on a shared production node. They require a disposable worker or nested VM +in the A3S OS environment. + +## 13. Security and Adversarial Testing + +- Fuzz every public JSON decoder, validation function, digest function, state + record decoder, log cursor decoder, and provider artifact URI adapter. +- Add property tests for validation boundaries and transition invariants. +- Run contract-only tests under Miri where supported and use a concurrency + model checker for any in-memory coordinator introduced later. +- Verify state path confinement against symlink replacement and time-of-check/ + time-of-use attempts from another process. +- Verify provider namespaces cannot discover, stop, remove, log, or exec units + owned by another namespace or node. +- Verify error, audit, evidence, and log artifacts never contain secret values + or registry credentials. +- Verify OCI digest pinning after pull and before create. +- Run dependency license, advisory, and supply-chain policy checks as a release + input; they supplement rather than replace behavioral tests. + +Fuzz regressions are checked in as minimal fixtures. Release candidates require +a bounded continuous fuzz run for every target and zero unresolved crashes, +panics, hangs, or uncontrolled allocations. + +## 14. Performance and Capacity Plan + +### 14.1 Core benchmarks + +Measure minimum, typical, and maximum-size records for: + +- specification validation and digest calculation; +- capability matching; +- observation and transition validation; +- state reserve, load, update, removal, and exact replay; +- records containing 1, 100, 1,000, and 10,000 receipts; +- same-unit contention and different-unit throughput at 1, 8, 32, and 128 + clients. + +### 14.2 Provider benchmarks + +Measure cold-image and warm-image Task apply, Service apply-to-healthy, +inspect, stop, remove, log pagination, provider restart recovery, and external +loss recovery. Record p50, p95, p99, throughput, CPU, RSS, file descriptors, +disk bytes, network bytes, and resource cleanup latency. + +### 14.3 Initial gates + +- Correctness counters allow zero lost receipts, duplicate resources, corrupt + records, unexplained log gaps, or leaked resources. +- A pull request fails when a controlled benchmark shows a statistically + significant regression greater than 10% in median or 15% in p95 against the + same runner class. +- Absolute latency SLOs are frozen only after three clean baselines on the + production-equivalent runner; they must not be guessed from laptop results. +- A 10,000-cycle churn test must finish with zero provider resources, mounts, + ports, state roots outside retention, and a non-growing file descriptor + count. +- A 24-hour merge soak and 72-hour release soak must show no monotonic RSS, + state-temporary-file, or provider-resource growth after workload count + returns to baseline. + +## 15. A3S OS Execution Safety + +### 15.1 Git-only deployment + +The runner checks out exact commits without uploading source: + +```text +git fetch --prune origin +git worktree add --detach /var/tmp/a3s-runtime-tests/ +``` + +Provider and consumer repositories use their own exact SHAs. The evidence +manifest records all of them. A dirty worktree is rejected. + +### 15.2 Isolation + +Every run has: + +- a globally unique run ID and provider namespace; +- a dedicated state root and evidence directory; +- pinned fixture image digests; +- loopback-only dynamic service ports; +- CPU, memory, PIDs, disk, duration, and concurrency limits; +- a process-level cleanup trap and an independent TTL sweeper; +- pre-run and post-run inventories keyed by managed labels. + +Shared production nodes run only L4 canaries: one bounded Task, one loopback +Service, inspect, logs when supported, stop, and remove. They never restart a +daemon, reboot the host, corrupt state, fill disk, alter firewall policy, use +production secrets, or mount production data. + +L3 and destructive L4 work runs on a disposable A3S OS worker that matches the +production kernel, architecture, provider version, filesystem, and security +policy. + +### 15.3 Preflight and stop conditions + +Preflight records host identity, kernel, architecture, cgroup mode, filesystem, +provider build, available CPU/memory/disk/inodes, existing managed resources, +and active deployments. The run does not start without the configured safety +headroom. + +Abort immediately on namespace collision, unexpected non-test resource +selection, failed cleanup, host free-space/inode threshold, sustained host load +threshold, production service error-budget burn, or a provider invariant +violation. Cleanup and evidence capture still run after an abort. + +## 16. Evidence Bundle + +Each run produces one immutable bundle: + +```text +evidence// +├── manifest.json +├── environment.json +├── preflight.json +├── junit.xml +├── coverage/ +├── benchmarks/ +├── faults/ +├── provider-inventory-before.json +├── provider-inventory-after.json +├── logs/ +└── cleanup.json +``` + +`manifest.json` contains run ID, timestamps, trigger, Runtime/provider/consumer +SHAs, fixture digests, selected layers and case IDs, configuration digest, and +final status. `cleanup.json` lists every created resource and proves removal or +an explicitly retained diagnostic artifact. Evidence is redacted before +upload, checksummed, retained by release policy, and linked from the commit or +release check. + +## 17. Delivery Phases + +### P0: Deterministic core gate + +- Resolve the contract decisions in Section 5. +- Add Linux CI for format, lint, docs, unit, integration, and compatibility + fixtures. +- Declare and test the minimum supported Rust version and current stable. +- Add shared fixture builders, stable case IDs, golden JSON, property tests, + and transition-matrix generation. +- Add a deterministic driver with call tracing and failures before and after + every provider boundary. +- Make skipped mandatory prerequisites fail their owning job. + +Exit: L0 and L1 are required checks, all protocol/lifecycle rows have an +identified test, and no unresolved P0 semantic decision remains. + +### P1: Durable recovery gate + +- Add multi-process state contention, failpoint, process-kill, permission, + corrupt-state, `ENOSPC`, and restart tests. +- Implement and verify the chosen generation and concurrent-operation model. +- Define and test receipt retention/compaction. + +Exit: every crash point preserves a valid durable state and produces no +untracked provider mutation in the deterministic provider model. + +### P2: Real Docker certification + +- Expand conformance into the profiles in Section 9. +- Run the mandatory Docker job on a dedicated Linux runner with the gate + explicitly enabled. +- Add Docker fault, resource, log, namespace, and leak cases. +- Run A3S Cloud consumer restart and reconciliation flows. + +Exit: every advertised Docker capability has passing real-provider evidence; +Base and Recovery pass; pre/post resource deltas are zero. + +### P3: Performance, adversarial, and soak gate + +- Add stable benchmark runners, fuzz targets, security cases, 10,000-cycle + churn, and 24/72-hour soak automation. +- Establish three clean baselines and freeze provider-specific SLOs. + +Exit: all correctness, regression, leak, and soak gates in Section 14 pass. + +### P4: Additional provider certification and production canary + +- For each new driver, supply fixtures for every advertised conformance + profile. +- Add A3S Box only after its `RuntimeDriver` exists and verify the `sandbox` + isolation mapping on production-equivalent Linux. +- Run the bounded L4 canary and verify cleanup. + +Exit: the provider matrix is evidence-backed for the release SHAs, and the +production canary bundle proves zero leaked resources and no safety stop. + +## 18. Release Gates + +A Runtime release is blocked unless: + +1. L0 and L1 pass for the exact release commit. +2. Golden compatibility changes have explicit schema approval. +3. The transition, request replay, generation, deadline, and state durability + matrices have no missing P0 row. +4. Base and Recovery pass for every provider declared production-supported. +5. Every advertised optional capability has passing profile evidence. +6. Required consumer end-to-end flows pass at pinned consumer commits. +7. Performance regression, resource leak, fuzz, and soak gates pass at the + release level required by the change risk. +8. Provider inventory after the final run matches the pre-run baseline. +9. The evidence bundle is complete, redacted, checksummed, and reviewable. + +Passing a mock-only suite, an environment-gated test that did not execute, or a +narrow provider smoke test is not sufficient evidence for a release claim. diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md new file mode 100644 index 0000000..28a222a --- /dev/null +++ b/docs/implementation-plan.md @@ -0,0 +1,162 @@ +# A3S Runtime Implementation Plan + +## Goal + +Implement the complete contract accepted by ADR 0001 and ADR 0002, certify all +advertised provider capabilities, and satisfy the release gates in the deep +test plan. Work is ordered by dependency; a later task cannot claim completion +from mocks when its required real-provider evidence is absent. + +## Working rules + +- Preserve user changes and keep each task reviewable. +- Write or update tests before completing the corresponding implementation. +- Run Cargo, provider, fault, performance, and soak validation only on Linux CI + or A3S OS. Do not run those workloads on a developer laptop. +- A3S OS obtains every repository through Git at an exact commit. +- Shared production nodes run only the bounded canary defined by the deep test + plan. Destructive tests require a disposable production-equivalent worker. +- Code and documentation are English, and product configuration remains ACL. + +## Task graph + +| ID | Task | Depends on | Completion evidence | +| --- | --- | --- | --- | +| R00 | Freeze baseline and task manifest | None | Exact Runtime and consumer SHAs, dirty-file ownership, and capability inventory recorded | +| R01 | Version every top-level wire record | R00 | Golden JSON for valid/old/future/unknown-field cases; Cloud wire consumers compile | +| R02 | Unify and bind provider identity | R01 | Typed capability ID, driver/factory checks, mismatch tests, Docker migration | +| R03 | Complete output and confidential capability matching | R01 | Output/attestation rejection before state work; exact output validation tests | +| R04 | Define and enforce operation postconditions | R01 | Full Task/Service result-state matrix; log and exec state tests | +| R05 | Bound deadlines across queue and dispatch | R04 | Deterministic clock tests for pre-lock, post-lock, provider timeout, and pending replay | +| R06 | Add per-unit cross-process operation leases | R04 | Same-unit race serialization and different-unit parallelism in independent processes | +| R07 | Replace embedded receipts with request journal v2 | R06 | Atomic receipt tests, restart replay, 10,001 requests, permissions, corruption handling | +| R08 | Make exec durably idempotent | R07 | Exact replay executes once; conflict, cancellation, timeout, and large-output tests | +| R09 | Enforce generation reconciliation | R06 | N to N+1 success/failure/crash tests with exactly one final provider resource | +| R10 | Build deterministic fault driver and transition generator | R01-R09 | Every state edge and provider boundary has a stable case ID and oracle | +| R11 | Split shared conformance into capability profiles | R10 | Base/Recovery mandatory; optional advertised profiles auto-run and cannot silently skip | +| R12 | Add Runtime repository CI and compatibility gate | R01-R11 | Required Linux MSRV/stable format, lint, docs, unit, integration, and golden checks | +| R13 | Certify Docker identity, lifecycle, and generation handling | R09-R12 | Real Docker Base and Recovery profiles pass; zero duplicate/leaked containers | +| R14 | Certify Docker network, mounts, health, resources, and logs | R11-R13 | Every advertised Docker capability has inspect plus behavioral evidence | +| R15 | Complete A3S Cloud consumer recovery paths | R13-R14 | Projection, journal, restart, redelivery, reconciliation, logs, cancel, and cleanup E2E pass | +| R16 | Implement A3S Box RuntimeDriver | R11 | Typed `sandbox` capability mapping with no provider-specific public fields | +| R17 | Certify A3S Box profiles | R16 | Base/Recovery and every advertised Box profile pass on production-equivalent Linux | +| R18 | Add fuzz, property, and security campaigns | R10-R17 | All declared targets run, regressions checked in, zero unresolved crash/hang/leak | +| R19 | Add benchmarks and regression budgets | R10-R17 | Three clean same-runner baselines and enforced median/p95 budgets | +| R20 | Add churn, fault, and 24/72-hour soak automation | R13-R19 | Evidence bundles prove zero resource/FD/state growth and all stop conditions | +| R21 | Run bounded A3S OS production canary | R12-R20 | Git-only exact-SHA canary passes with complete cleanup and no safety stop | +| R22 | Perform release completion audit | R01-R21 | Every deep-test-plan release gate maps to authoritative passing evidence | + +## Work packages + +### Package A: Protocol completeness (`R00`-`R05`) + +Deliverables: + +- ADR 0002 implementation; +- schema constants and fields for inspection, log, and exec surfaces; +- `ProviderId` as the sole provider identity type; +- output artifact sizes and `OutputArtifacts` capability; +- confidential-attestation capability matching; +- explicit apply, stop, logs, and exec state guards; +- deadline wrappers with deterministic tests; +- golden wire fixtures and schema compatibility tests; +- declared Rust MSRV. + +Exit gate: every public wire value has an explicit version and every operation +has a deterministic state/deadline oracle. + +### Package B: Durable coordination (`R06`-`R10`) + +Deliverables: + +- independent operation and record locks; +- v2 unit directory and request journal; +- durable exec receipts; +- generation reconciliation contract and deterministic provider model; +- process-level crash/failpoint harness; +- transition matrix and race matrix; +- hostile filesystem, permission, corruption, disk-full, and cancellation + cases. + +Exit gate: every mutating crash point can replay after a new process starts, +with valid state and no untracked provider resource. + +### Package C: Shared certification (`R11`-`R12`) + +Deliverables: + +- composable Base, Recovery, Networking, Mounts, Health, Resources, Logs, Exec, + Security, Outputs, and Evidence conformance profiles; +- fixture traits and cleanup inventory contracts; +- CI jobs that fail when a mandatory provider prerequisite is missing; +- MSRV and stable validation, documentation, golden compatibility, and + cross-architecture digest evidence. + +Exit gate: mock and file-state correctness is independently reproducible for an +exact commit on Linux CI. + +### Package D: Docker and A3S Cloud (`R13`-`R15`) + +Deliverables: + +- stale-generation container discovery and cleanup; +- provider identity and label-tamper enforcement; +- Task success/failure/timeout and Service health profiles; +- network, volume, tmpfs, CPU, memory, PIDs, restart, and log-cursor probes; +- node-agent/Docker restart and external-deletion recovery; +- Cloud projection, command journal, redelivery, reconciliation, cancellation, + log transport, and cleanup E2E tests. + +Exit gate: real Docker and Cloud evidence covers every source-advertised +capability, and post-run provider inventory equals its baseline. + +### Package E: A3S Box (`R16`-`R17`) + +Deliverables: + +- a driver hosted at the provider integration boundary; +- digest-pinned OCI apply/inspect/stop/remove/logs/exec mapping; +- stable unit/generation/request labels or metadata; +- `IsolationLevel::Sandbox` mapping; +- output, usage, attestation, secret, network, mount, and resource claims only + when implemented and tested; +- crash, shim loss, VM loss, host restart, and cleanup profiles. + +Exit gate: A3S Box passes Base and Recovery plus every capability it advertises +on production-equivalent A3S OS Linux. + +### Package F: Non-functional release proof (`R18`-`R22`) + +Deliverables: + +- fuzz/property/security campaigns; +- controlled core and provider benchmarks; +- 10,000-cycle churn, resource leak detection, fault automation, and 24/72-hour + soak; +- immutable evidence bundles; +- bounded production canary; +- requirement-by-requirement release audit. + +Exit gate: all nine release gates in the deep test plan have direct, +reviewable, exact-SHA evidence. + +## Immediate execution order + +1. Preserve the existing optional ephemeral-storage and `unknown` recovery + changes as inputs to ADR 0002. +2. Implement `R01`-`R04` with contract tests and update A3S Cloud constructors + in the same compatibility change. +3. Push an exact feature commit so the A3S OS runner can build and test through + Git; do not upload a working tree archive. +4. Implement `R05`-`R10`, rerunning the server-side core matrix after each + durable-state slice. +5. Enable and expand real Docker conformance before starting the A3S Box + adapter, so the shared provider oracles are proven by one real driver first. + +## Completion rule + +The project is complete only when `R22` can map every explicit invariant, +profile, performance gate, soak gate, production-safety condition, and cleanup +requirement to authoritative evidence. An unexecuted environment-gated test, +mock-only result, intended provider feature, or passing narrow smoke test is not +completion evidence. diff --git a/src/conformance.rs b/src/conformance.rs index 400de0d..b8fc2e7 100644 --- a/src/conformance.rs +++ b/src/conformance.rs @@ -176,7 +176,7 @@ fn validate_action( fn require_found(label: &str, inspection: RuntimeInspection) -> RuntimeResult { match inspection { - RuntimeInspection::Found { observation } => Ok(*observation), + RuntimeInspection::Found { observation, .. } => Ok(*observation), RuntimeInspection::NotFound { .. } => Err(RuntimeError::Protocol(format!( "{label} unexpectedly returned not found" ))), diff --git a/src/contract/artifact.rs b/src/contract/artifact.rs index 68c1890..103a4e2 100644 --- a/src/contract/artifact.rs +++ b/src/contract/artifact.rs @@ -26,6 +26,7 @@ impl ArtifactRef { pub struct RuntimeOutputArtifact { pub name: String, pub artifact: ArtifactRef, + pub size_bytes: u64, } impl RuntimeOutputArtifact { diff --git a/src/contract/capabilities.rs b/src/contract/capabilities.rs index 4381618..3668d6f 100644 --- a/src/contract/capabilities.rs +++ b/src/contract/capabilities.rs @@ -1,6 +1,7 @@ use super::{ HealthCheckKind, IsolationLevel, MountKind, NetworkMode, RuntimeUnitClass, RuntimeUnitSpec, }; +use crate::ProviderId; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; @@ -25,6 +26,7 @@ pub enum RuntimeFeature { Usage, Attestation, SecretReferences, + OutputArtifacts, } /// Structured, provider-reported capabilities. Product-specific support @@ -33,7 +35,7 @@ pub enum RuntimeFeature { #[serde(deny_unknown_fields)] pub struct RuntimeCapabilities { pub schema: String, - pub provider_id: String, + pub provider_id: ProviderId, pub provider_build: String, pub unit_classes: Vec, pub artifact_media_types: Vec, @@ -46,7 +48,7 @@ pub struct RuntimeCapabilities { } impl RuntimeCapabilities { - pub const SCHEMA: &'static str = "a3s.runtime.capabilities.v2"; + pub const SCHEMA: &'static str = "a3s.runtime.capabilities.v3"; pub fn validate(&self) -> Result<(), String> { if self.schema != Self::SCHEMA { @@ -55,7 +57,6 @@ impl RuntimeCapabilities { self.schema )); } - super::validate_id("provider_id", &self.provider_id, 64)?; super::validate_nonempty("provider_build", &self.provider_build, 255)?; if self.unit_classes.is_empty() || self.artifact_media_types.is_empty() @@ -116,12 +117,18 @@ impl RuntimeCapabilities { ResourceControl::Cpu, ResourceControl::Memory, ResourceControl::Pids, - ResourceControl::EphemeralStorage, ] { if !self.resource_controls.contains(&required) { missing.push(format!("resource_control:{required:?}")); } } + if spec.resources.ephemeral_storage_bytes.is_some() + && !self + .resource_controls + .contains(&ResourceControl::EphemeralStorage) + { + missing.push("resource_control:EphemeralStorage".into()); + } if spec.resources.execution_timeout_ms.is_some() && !self .resource_controls @@ -135,6 +142,14 @@ impl RuntimeCapabilities { if !spec.secrets.is_empty() && !self.supports_feature(RuntimeFeature::SecretReferences) { missing.push("feature:SecretReferences".into()); } + if !spec.outputs.is_empty() && !self.supports_feature(RuntimeFeature::OutputArtifacts) { + missing.push("feature:OutputArtifacts".into()); + } + if spec.isolation == IsolationLevel::Confidential + && !self.supports_feature(RuntimeFeature::Attestation) + { + missing.push("feature:Attestation".into()); + } missing.sort(); missing.dedup(); Ok(missing) diff --git a/src/contract/observation.rs b/src/contract/observation.rs index 58c886f..66bde87 100644 --- a/src/contract/observation.rs +++ b/src/contract/observation.rs @@ -1,4 +1,6 @@ -use super::{ArtifactRef, RuntimeOutputArtifact, RuntimeUnitClass, RuntimeUnitSpec}; +use super::{ + ArtifactRef, IsolationLevel, RuntimeOutputArtifact, RuntimeUnitClass, RuntimeUnitSpec, +}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; @@ -117,7 +119,7 @@ pub struct RuntimeObservation { } impl RuntimeObservation { - pub const SCHEMA: &'static str = "a3s.runtime.observation.v1"; + pub const SCHEMA: &'static str = "a3s.runtime.observation.v2"; pub(crate) fn accepted(spec: &RuntimeUnitSpec, observed_at_ms: u64) -> Result { Ok(Self { @@ -227,6 +229,40 @@ impl RuntimeObservation { { return Err("Runtime observation does not match the unit specification".into()); } + if self.state == RuntimeUnitState::Succeeded { + if self.outputs.len() != spec.outputs.len() { + return Err("succeeded Task did not report the exact requested outputs".into()); + } + for expected in &spec.outputs { + let output = self + .outputs + .iter() + .find(|output| output.name == expected.name) + .ok_or_else(|| format!("succeeded Task omitted output {:?}", expected.name))?; + if output.artifact.media_type != expected.media_type { + return Err(format!( + "output {:?} media type does not match its specification", + expected.name + )); + } + if output.size_bytes > expected.max_bytes { + return Err(format!( + "output {:?} exceeds its maximum size", + expected.name + )); + } + } + } else if !self.outputs.is_empty() { + return Err("only a succeeded Task may report outputs".into()); + } + if spec.isolation == IsolationLevel::Confidential + && self.provider_resource_id.is_some() + && self.provider_attestation.is_none() + { + return Err( + "provider-backed confidential Runtime observation requires attestation".into(), + ); + } Ok(()) } @@ -252,22 +288,34 @@ impl RuntimeObservation { #[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] pub enum RuntimeInspection { Found { + schema: String, observation: Box, }, NotFound { + schema: String, unit_id: String, last_generation: Option, }, } impl RuntimeInspection { + pub const SCHEMA: &'static str = "a3s.runtime.inspection.v1"; + pub fn validate(&self) -> Result<(), String> { match self { - Self::Found { observation } => observation.validate(), + Self::Found { + schema, + observation, + } => { + validate_inspection_schema(schema)?; + observation.validate() + } Self::NotFound { + schema, unit_id, last_generation, } => { + validate_inspection_schema(schema)?; super::validate_id("unit_id", unit_id, 512)?; if *last_generation == Some(0) { return Err("last_generation must be positive when present".into()); @@ -277,3 +325,10 @@ impl RuntimeInspection { } } } + +fn validate_inspection_schema(schema: &str) -> Result<(), String> { + if schema != RuntimeInspection::SCHEMA { + return Err(format!("unsupported Runtime inspection schema {schema:?}")); + } + Ok(()) +} diff --git a/src/contract/protocol.rs b/src/contract/protocol.rs index 534db8b..a5340e6 100644 --- a/src/contract/protocol.rs +++ b/src/contract/protocol.rs @@ -106,6 +106,7 @@ pub enum RuntimeLogStream { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeLogQuery { + pub schema: String, pub unit_id: String, pub generation: u64, pub cursor: Option, @@ -114,7 +115,15 @@ pub struct RuntimeLogQuery { } impl RuntimeLogQuery { + pub const SCHEMA: &'static str = "a3s.runtime.log-query.v1"; + pub fn validate(&self) -> Result<(), String> { + if self.schema != Self::SCHEMA { + return Err(format!( + "unsupported Runtime log query schema {:?}", + self.schema + )); + } super::validate_id("unit_id", &self.unit_id, 512)?; if self.generation == 0 || self.limit == 0 || self.limit > 10_000 { return Err("log generation or limit is invalid".into()); @@ -133,6 +142,7 @@ impl RuntimeLogQuery { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeLogChunk { + pub schema: String, pub cursor: String, pub sequence: u64, pub observed_at_ms: u64, @@ -141,7 +151,15 @@ pub struct RuntimeLogChunk { } impl RuntimeLogChunk { + pub const SCHEMA: &'static str = "a3s.runtime.log-chunk.v1"; + pub fn validate(&self) -> Result<(), String> { + if self.schema != Self::SCHEMA { + return Err(format!( + "unsupported Runtime log chunk schema {:?}", + self.schema + )); + } super::validate_nonempty("log cursor", &self.cursor, 1024)?; if self.data.len() > 1024 * 1024 { return Err("log chunk exceeds one MiB".into()); @@ -153,19 +171,30 @@ impl RuntimeLogChunk { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeExecRequest { + pub schema: String, pub request_id: String, pub unit_id: String, pub generation: u64, pub command: Vec, pub timeout_ms: u64, + pub deadline_at_ms: Option, } impl RuntimeExecRequest { + pub const SCHEMA: &'static str = "a3s.runtime.exec-request.v1"; + pub fn validate(&self) -> Result<(), String> { + if self.schema != Self::SCHEMA { + return Err(format!( + "unsupported Runtime exec request schema {:?}", + self.schema + )); + } super::validate_id("request_id", &self.request_id, 512)?; super::validate_id("unit_id", &self.unit_id, 512)?; if self.generation == 0 || self.timeout_ms == 0 + || self.deadline_at_ms == Some(0) || self.command.is_empty() || self.command.len() > 256 || self @@ -177,11 +206,16 @@ impl RuntimeExecRequest { } Ok(()) } + + pub fn digest(&self) -> Result { + canonical_digest(self, self.validate()) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeExecResult { + pub schema: String, pub request_id: String, pub observation: RuntimeObservation, pub exit_code: i32, @@ -191,7 +225,15 @@ pub struct RuntimeExecResult { } impl RuntimeExecResult { + pub const SCHEMA: &'static str = "a3s.runtime.exec-result.v1"; + pub fn validate(&self) -> Result<(), String> { + if self.schema != Self::SCHEMA { + return Err(format!( + "unsupported Runtime exec result schema {:?}", + self.schema + )); + } super::validate_id("request_id", &self.request_id, 512)?; self.observation.validate()?; if self.stdout.len() > 16 * 1024 * 1024 || self.stderr.len() > 16 * 1024 * 1024 { diff --git a/src/contract/resource.rs b/src/contract/resource.rs index 7e10ffd..36d5972 100644 --- a/src/contract/resource.rs +++ b/src/contract/resource.rs @@ -15,19 +15,20 @@ pub struct ResourceLimits { pub cpu_millis: u64, pub memory_bytes: u64, pub pids: u32, - pub ephemeral_storage_bytes: u64, + /// Optional writable-layer quota. Providers must advertise + /// `ResourceControl::EphemeralStorage` before accepting this limit. + pub ephemeral_storage_bytes: Option, /// Required for finite Tasks and forbidden for long-running Services. pub execution_timeout_ms: Option, } impl ResourceLimits { pub(crate) fn validate(&self) -> Result<(), String> { - if self.cpu_millis == 0 - || self.memory_bytes == 0 - || self.pids == 0 - || self.ephemeral_storage_bytes == 0 - { - return Err("all Runtime resource limits must be positive".into()); + if self.cpu_millis == 0 || self.memory_bytes == 0 || self.pids == 0 { + return Err("required Runtime resource limits must be positive".into()); + } + if self.ephemeral_storage_bytes == Some(0) { + return Err("ephemeral_storage_bytes must be positive when present".into()); } if self.execution_timeout_ms == Some(0) { return Err("execution_timeout_ms must be positive when present".into()); diff --git a/src/contract/unit.rs b/src/contract/unit.rs index f260b6c..e335f49 100644 --- a/src/contract/unit.rs +++ b/src/contract/unit.rs @@ -324,7 +324,7 @@ mod tests { cpu_millis: 500, memory_bytes: 128 * 1024 * 1024, pids: 128, - ephemeral_storage_bytes: 1024 * 1024 * 1024, + ephemeral_storage_bytes: Some(1024 * 1024 * 1024), execution_timeout_ms: timeout, } } diff --git a/src/driver.rs b/src/driver.rs index f4a28b7..284fb26 100644 --- a/src/driver.rs +++ b/src/driver.rs @@ -3,7 +3,7 @@ use crate::contract::{ RuntimeInspection, RuntimeLogChunk, RuntimeLogQuery, RuntimeObservation, RuntimeRemoval, RuntimeUnitSpec, }; -use crate::{RuntimeResult, RuntimeUnitRecord}; +use crate::{ProviderId, RuntimeResult, RuntimeUnitRecord}; use async_trait::async_trait; /// Provider-specific primitive used by `ManagedRuntimeClient`. @@ -13,6 +13,8 @@ use async_trait::async_trait; /// ambiguous transport failure using the stable unit identity and generation. #[async_trait] pub trait RuntimeDriver: Send + Sync { + fn provider_id(&self) -> &ProviderId; + async fn capabilities(&self) -> RuntimeResult; async fn apply( diff --git a/src/managed.rs b/src/managed.rs index 1a26fb9..6612e62 100644 --- a/src/managed.rs +++ b/src/managed.rs @@ -37,6 +37,13 @@ impl ManagedRuntimeClient { async fn checked_capabilities(&self) -> RuntimeResult { let capabilities = self.driver.capabilities().await?; capabilities.validate().map_err(RuntimeError::Protocol)?; + if &capabilities.provider_id != self.driver.provider_id() { + return Err(RuntimeError::Protocol(format!( + "Runtime driver {:?} reported capabilities for {:?}", + self.driver.provider_id().as_str(), + capabilities.provider_id.as_str() + ))); + } Ok(capabilities) } @@ -84,11 +91,7 @@ impl RuntimeClient for ManagedRuntimeClient { observation .validate_against(&request.spec) .map_err(RuntimeError::Protocol)?; - if observation.state == RuntimeUnitState::Accepted { - return Err(RuntimeError::Protocol( - "provider apply did not advance the accepted observation".into(), - )); - } + ensure_apply_result(&request.spec, &observation)?; Ok(self .state .update_observation(Some(&request.request_id), &observation) @@ -101,6 +104,7 @@ impl RuntimeClient for ManagedRuntimeClient { Ok(record) => record, Err(RuntimeError::NotFound { .. }) => { return Ok(RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), unit_id: unit_id.into(), last_generation: None, }); @@ -109,18 +113,22 @@ impl RuntimeClient for ManagedRuntimeClient { }; if record.removed_at_ms.is_some() { return Ok(RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), unit_id: unit_id.into(), last_generation: Some(record.spec.generation), }); } if record.observation.state.is_terminal() { return Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(record.observation), }); } - match self.driver.inspect(&record).await? { - RuntimeInspection::Found { observation } => { + let inspection = self.driver.inspect(&record).await?; + inspection.validate().map_err(RuntimeError::Protocol)?; + match inspection { + RuntimeInspection::Found { observation, .. } => { observation .validate_against(&record.spec) .map_err(RuntimeError::Protocol)?; @@ -129,6 +137,7 @@ impl RuntimeClient for ManagedRuntimeClient { .update_observation(None, observation.as_ref()) .await?; Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(record.observation), }) } @@ -142,6 +151,7 @@ impl RuntimeClient for ManagedRuntimeClient { unknown.failure = None; let record = self.state.update_observation(None, &unknown).await?; Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(record.observation), }) } @@ -163,6 +173,7 @@ impl RuntimeClient for ManagedRuntimeClient { .await?; if !reservation.dispatch { return Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(reservation.receipt.observation.ok_or_else(|| { RuntimeError::Protocol("completed stop receipt has no observation".into()) })?), @@ -172,11 +183,13 @@ impl RuntimeClient for ManagedRuntimeClient { observation .validate_against(&reservation.record.spec) .map_err(RuntimeError::Protocol)?; + ensure_stop_result(&reservation.record.observation, &observation)?; let record = self .state .update_observation(Some(&request.request_id), &observation) .await?; Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(record.observation), }) } @@ -222,7 +235,7 @@ impl RuntimeClient for ManagedRuntimeClient { ])); } let record = self.state.load(&query.unit_id).await?; - ensure_active_generation(&record, query.generation)?; + ensure_current_generation(&record, query.generation)?; let chunks = self.driver.logs(&record, query).await?; for chunk in &chunks { chunk.validate().map_err(RuntimeError::Protocol)?; @@ -247,9 +260,19 @@ impl RuntimeClient for ManagedRuntimeClient { ])); } let record = self.state.load(&request.unit_id).await?; - ensure_active_generation(&record, request.generation)?; + ensure_current_generation(&record, request.generation)?; + if record.observation.state != RuntimeUnitState::Running { + return Err(RuntimeError::InvalidRequest(format!( + "Runtime exec requires a running unit; {:?} is {:?}", + request.unit_id, record.observation.state + ))); + } let result = self.driver.exec(&record, request).await?; result.validate().map_err(RuntimeError::Protocol)?; + result + .observation + .validate_against(&record.spec) + .map_err(RuntimeError::Protocol)?; if result.request_id != request.request_id || result.observation.unit_id != request.unit_id || result.observation.generation != request.generation @@ -262,7 +285,7 @@ impl RuntimeClient for ManagedRuntimeClient { } } -fn ensure_active_generation( +fn ensure_current_generation( record: &crate::RuntimeUnitRecord, requested: u64, ) -> RuntimeResult<()> { @@ -286,3 +309,50 @@ fn ensure_active_generation( } Ok(()) } + +fn ensure_apply_result( + spec: &crate::contract::RuntimeUnitSpec, + observation: &RuntimeObservation, +) -> RuntimeResult<()> { + let allowed = match spec.class { + crate::contract::RuntimeUnitClass::Task => matches!( + observation.state, + RuntimeUnitState::Succeeded | RuntimeUnitState::Failed + ), + crate::contract::RuntimeUnitClass::Service => matches!( + observation.state, + RuntimeUnitState::Running + | RuntimeUnitState::Stopped + | RuntimeUnitState::Failed + | RuntimeUnitState::Unknown + ), + }; + if !allowed { + return Err(RuntimeError::Protocol(format!( + "provider apply returned invalid {:?} result {:?}", + spec.class, observation.state + ))); + } + if observation.provider_resource_id.is_none() || observation.provider_build.is_none() { + return Err(RuntimeError::Protocol( + "provider apply returned an observation without provider identity".into(), + )); + } + Ok(()) +} + +fn ensure_stop_result( + current: &RuntimeObservation, + observation: &RuntimeObservation, +) -> RuntimeResult<()> { + if observation.state == RuntimeUnitState::Stopped + || observation.state == RuntimeUnitState::Unknown + || current.state.is_terminal() && observation == current + { + return Ok(()); + } + Err(RuntimeError::Protocol(format!( + "provider stop returned nonterminal state {:?}", + observation.state + ))) +} diff --git a/src/registry.rs b/src/registry.rs index b66a4c2..9f6365d 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -1,12 +1,14 @@ use crate::{ProviderId, RuntimeClient, RuntimeError, RuntimeResult}; +use async_trait::async_trait; use std::collections::BTreeMap; use std::sync::Arc; /// Typed construction boundary for one Runtime provider implementation. +#[async_trait] pub trait RuntimeProviderFactory: Send + Sync { fn provider_id(&self) -> &ProviderId; - fn create(&self) -> RuntimeResult>; + async fn create(&self) -> RuntimeResult>; } /// Registry of provider factories. Selection policy belongs to the caller; @@ -36,8 +38,9 @@ impl RuntimeClientRegistry { self.factories.contains_key(provider) } - pub fn connect(&self, provider: &ProviderId) -> RuntimeResult> { - self.factories + pub async fn connect(&self, provider: &ProviderId) -> RuntimeResult> { + let client = self + .factories .get(provider) .ok_or_else(|| { RuntimeError::ProviderUnavailable(format!( @@ -46,5 +49,16 @@ impl RuntimeClientRegistry { )) })? .create() + .await?; + let capabilities = client.capabilities().await?; + capabilities.validate().map_err(RuntimeError::Protocol)?; + if &capabilities.provider_id != provider { + return Err(RuntimeError::Protocol(format!( + "Runtime provider factory {:?} created client reporting {:?}", + provider.as_str(), + capabilities.provider_id.as_str() + ))); + } + Ok(client) } } diff --git a/src/state/file.rs b/src/state/file.rs index e045ec2..5523a63 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -70,7 +70,10 @@ impl FileRuntimeStateStore { } let mut receipt = RuntimeRequestReceipt::pending_apply(&request).map_err(RuntimeError::Protocol)?; - if record.observation.state != RuntimeUnitState::Accepted { + if !matches!( + record.observation.state, + RuntimeUnitState::Accepted | RuntimeUnitState::Unknown + ) { receipt.complete_with_observation(record.observation.clone()); } record.requests.insert(request.request_id.clone(), receipt); @@ -352,7 +355,8 @@ pub(crate) fn validate_transition( .map_err(RuntimeError::Protocol)?; next.validate_against(spec) .map_err(RuntimeError::Protocol)?; - if current.provider_resource_id.is_some() + if current.state != RuntimeUnitState::Unknown + && current.provider_resource_id.is_some() && current.provider_resource_id != next.provider_resource_id { return Err(RuntimeError::Protocol( diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index a1cabdd..6cfc4ad 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -3,9 +3,9 @@ use a3s_runtime::contract::{ ResourceControl, ResourceLimits, RestartPolicy, RuntimeActionRequest, RuntimeApplyRequest, RuntimeCapabilities, RuntimeExecRequest, RuntimeExecResult, RuntimeFeature, RuntimeHealthCheck, RuntimeHealthObservation, RuntimeHealthState, RuntimeInspection, RuntimeLogChunk, - RuntimeLogQuery, RuntimeLogStream, RuntimeNetworkSpec, RuntimeObservation, RuntimePort, - RuntimeProcessSpec, RuntimeRemoval, RuntimeUnitClass, RuntimeUnitSpec, RuntimeUnitState, - TransportProtocol, + RuntimeLogQuery, RuntimeLogStream, RuntimeNetworkSpec, RuntimeObservation, + RuntimeOutputArtifact, RuntimeOutputSpec, RuntimePort, RuntimeProcessSpec, RuntimeRemoval, + RuntimeUnitClass, RuntimeUnitSpec, RuntimeUnitState, TransportProtocol, }; use a3s_runtime::{ verify_runtime_provider, FileRuntimeStateStore, ManagedRuntimeClient, ProviderId, @@ -30,29 +30,42 @@ impl RuntimeClock for FixedClock { } struct TestDriver { + provider: ProviderId, capabilities: RuntimeCapabilities, apply_calls: AtomicUsize, inspect_calls: AtomicUsize, stop_calls: AtomicUsize, remove_calls: AtomicUsize, + exec_calls: AtomicUsize, fail_next_apply: AtomicBool, missing_on_inspect: AtomicBool, substitute_identity: AtomicBool, unordered_logs: AtomicBool, + return_starting_on_apply: AtomicBool, + return_unknown_on_apply: AtomicBool, + return_running_on_stop: AtomicBool, + substitute_exec_spec: AtomicBool, } impl TestDriver { fn new() -> Self { + let provider = ProviderId::parse("test-runtime").unwrap(); Self { - capabilities: capabilities(), + capabilities: capabilities(provider.clone()), + provider, apply_calls: AtomicUsize::new(0), inspect_calls: AtomicUsize::new(0), stop_calls: AtomicUsize::new(0), remove_calls: AtomicUsize::new(0), + exec_calls: AtomicUsize::new(0), fail_next_apply: AtomicBool::new(false), missing_on_inspect: AtomicBool::new(false), substitute_identity: AtomicBool::new(false), unordered_logs: AtomicBool::new(false), + return_starting_on_apply: AtomicBool::new(false), + return_unknown_on_apply: AtomicBool::new(false), + return_running_on_stop: AtomicBool::new(false), + substitute_exec_spec: AtomicBool::new(false), } } @@ -68,6 +81,10 @@ impl TestDriver { #[async_trait] impl RuntimeDriver for TestDriver { + fn provider_id(&self) -> &ProviderId { + &self.provider + } + async fn capabilities(&self) -> RuntimeResult { Ok(self.capabilities.clone()) } @@ -82,23 +99,55 @@ impl RuntimeDriver for TestDriver { return Err(RuntimeError::Transport("ambiguous apply".into())); } let mut observation = current.clone(); - observation.state = match spec.class { - RuntimeUnitClass::Task => RuntimeUnitState::Succeeded, - RuntimeUnitClass::Service => RuntimeUnitState::Running, + observation.state = if self.return_unknown_on_apply.load(Ordering::SeqCst) { + RuntimeUnitState::Unknown + } else if self.return_starting_on_apply.load(Ordering::SeqCst) { + RuntimeUnitState::Starting + } else { + match spec.class { + RuntimeUnitClass::Task => RuntimeUnitState::Succeeded, + RuntimeUnitClass::Service => RuntimeUnitState::Running, + } }; if self.substitute_identity.load(Ordering::SeqCst) { observation.unit_id = "substituted".into(); } - observation.provider_resource_id = Some(format!("provider/{}", spec.unit_id)); - observation.provider_build = Some("test-driver/1".into()); + if observation.state != RuntimeUnitState::Unknown + || !self.return_unknown_on_apply.load(Ordering::SeqCst) + { + let provider_identity = if current.state == RuntimeUnitState::Unknown { + format!("provider/recovered/{}", spec.unit_id) + } else { + format!("provider/{}", spec.unit_id) + }; + observation.provider_resource_id = Some(provider_identity); + observation.provider_build = Some("test-driver/1".into()); + } observation.observed_at_ms = NOW + 1; observation.started_at_ms = Some(NOW); - observation.finished_at_ms = (spec.class == RuntimeUnitClass::Task).then_some(NOW + 1); + observation.finished_at_ms = observation.state.is_terminal().then_some(NOW + 1); observation.health = spec.health.as_ref().map(|_| RuntimeHealthObservation { state: RuntimeHealthState::Healthy, checked_at_ms: NOW + 1, message: None, }); + observation.outputs = if observation.state == RuntimeUnitState::Succeeded { + spec.outputs + .iter() + .map(|expected| RuntimeOutputArtifact { + name: expected.name.clone(), + artifact: ArtifactRef { + media_type: expected.media_type.clone(), + ..artifact('b') + }, + size_bytes: expected.max_bytes.min(4), + }) + .collect() + } else { + Vec::new() + }; + observation.provider_attestation = + (spec.isolation == IsolationLevel::Confidential).then(|| artifact('c')); Ok(observation) } @@ -106,6 +155,7 @@ impl RuntimeDriver for TestDriver { self.inspect_calls.fetch_add(1, Ordering::SeqCst); if self.missing_on_inspect.load(Ordering::SeqCst) { return Ok(RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), unit_id: unit.spec.unit_id.clone(), last_generation: Some(unit.spec.generation), }); @@ -113,6 +163,7 @@ impl RuntimeDriver for TestDriver { let mut observation = unit.observation.clone(); observation.observed_at_ms += 1; Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(observation), }) } @@ -124,10 +175,19 @@ impl RuntimeDriver for TestDriver { ) -> RuntimeResult { self.stop_calls.fetch_add(1, Ordering::SeqCst); let mut observation = unit.observation.clone(); - observation.state = RuntimeUnitState::Stopped; + observation.state = if self.return_running_on_stop.load(Ordering::SeqCst) { + RuntimeUnitState::Running + } else { + RuntimeUnitState::Stopped + }; observation.observed_at_ms += 1; - observation.finished_at_ms = Some(observation.observed_at_ms); - observation.health = None; + observation.finished_at_ms = observation + .state + .is_terminal() + .then_some(observation.observed_at_ms); + if observation.state.is_terminal() { + observation.health = None; + } observation.outputs.clear(); Ok(observation) } @@ -160,6 +220,7 @@ impl RuntimeDriver for TestDriver { }; Ok(vec![ RuntimeLogChunk { + schema: RuntimeLogChunk::SCHEMA.into(), cursor: "cursor-1".into(), sequence: 1, observed_at_ms: NOW, @@ -167,6 +228,7 @@ impl RuntimeDriver for TestDriver { data: "started\n".into(), }, RuntimeLogChunk { + schema: RuntimeLogChunk::SCHEMA.into(), cursor: "cursor-2".into(), sequence: second_sequence, observed_at_ms: NOW + 1, @@ -181,9 +243,15 @@ impl RuntimeDriver for TestDriver { unit: &RuntimeUnitRecord, request: &RuntimeExecRequest, ) -> RuntimeResult { + self.exec_calls.fetch_add(1, Ordering::SeqCst); + let mut observation = unit.observation.clone(); + if self.substitute_exec_spec.load(Ordering::SeqCst) { + observation.spec_digest = digest('f'); + } Ok(RuntimeExecResult { + schema: RuntimeExecResult::SCHEMA.into(), request_id: request.request_id.clone(), - observation: unit.observation.clone(), + observation, exit_code: 0, stdout: "ok\n".into(), stderr: String::new(), @@ -212,7 +280,7 @@ fn resources(timeout: Option) -> ResourceLimits { cpu_millis: 500, memory_bytes: 128 * 1024 * 1024, pids: 128, - ephemeral_storage_bytes: 1024 * 1024 * 1024, + ephemeral_storage_bytes: Some(1024 * 1024 * 1024), execution_timeout_ms: timeout, } } @@ -292,10 +360,10 @@ fn action(request_id: &str, unit_id: &str, generation: u64) -> RuntimeActionRequ } } -fn capabilities() -> RuntimeCapabilities { +fn capabilities(provider_id: ProviderId) -> RuntimeCapabilities { RuntimeCapabilities { schema: RuntimeCapabilities::SCHEMA.into(), - provider_id: "test-runtime".into(), + provider_id, provider_build: "test-driver/1".into(), unit_classes: vec![RuntimeUnitClass::Task, RuntimeUnitClass::Service], artifact_media_types: vec![IMAGE_MEDIA_TYPE.into()], @@ -324,6 +392,7 @@ fn capabilities() -> RuntimeCapabilities { RuntimeFeature::Remove, RuntimeFeature::Logs, RuntimeFeature::Exec, + RuntimeFeature::OutputArtifacts, ], } } @@ -423,7 +492,7 @@ async fn stop_remove_and_absence_are_durable_and_idempotent() { let first_stop = client.stop(&stop).await.unwrap(); assert!(matches!( &first_stop, - RuntimeInspection::Found { observation } + RuntimeInspection::Found { observation, .. } if observation.state == RuntimeUnitState::Stopped )); assert_eq!(client.stop(&stop).await.unwrap(), first_stop); @@ -444,6 +513,7 @@ async fn stop_remove_and_absence_are_durable_and_idempotent() { assert_eq!( client.inspect("service-lifecycle").await.unwrap(), RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), unit_id: "service-lifecycle".into(), last_generation: Some(1), } @@ -469,6 +539,294 @@ async fn unsupported_capabilities_fail_before_state_or_provider_work() { assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 0); } +#[tokio::test] +async fn optional_resource_controls_gate_only_workloads_that_request_them() { + let mut driver = TestDriver::new(); + driver + .capabilities + .resource_controls + .retain(|control| *control != ResourceControl::EphemeralStorage); + let driver = Arc::new(driver); + let (client, store) = driver.client(); + + let mut without_quota = service("without-ephemeral-quota", 1); + without_quota.resources.ephemeral_storage_bytes = None; + client + .apply(&apply("without-ephemeral-quota", without_quota)) + .await + .expect("unrelated workload must remain supported"); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); + + assert!(matches!( + client + .apply(&apply( + "with-ephemeral-quota", + service("with-ephemeral-quota", 1), + )) + .await, + Err(RuntimeError::UnsupportedCapabilities(missing)) + if missing == vec!["resource_control:EphemeralStorage"] + )); + assert!(matches!( + store.load("with-ephemeral-quota").await, + Err(RuntimeError::NotFound { .. }) + )); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn provider_identity_mismatch_fails_before_state_or_provider_work() { + let mut driver = TestDriver::new(); + driver.capabilities.provider_id = ProviderId::parse("substituted-provider").unwrap(); + let driver = Arc::new(driver); + let (client, store) = driver.client(); + + assert!(matches!( + client + .apply(&apply("provider-mismatch", service("provider-mismatch", 1))) + .await, + Err(RuntimeError::Protocol(message)) if message.contains("reported capabilities") + )); + assert!(matches!( + store.load("provider-mismatch").await, + Err(RuntimeError::NotFound { .. }) + )); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn output_and_confidential_capabilities_fail_closed_and_outputs_are_exact() { + let mut output_task = task("output-task", 1); + output_task.outputs = vec![RuntimeOutputSpec { + name: "result".into(), + path: "/outputs/result.json".into(), + media_type: "application/json".into(), + max_bytes: 16, + }]; + + let mut unsupported = TestDriver::new(); + unsupported + .capabilities + .features + .retain(|feature| *feature != RuntimeFeature::OutputArtifacts); + let unsupported = Arc::new(unsupported); + let (client, store) = unsupported.client(); + assert!(matches!( + client + .apply(&apply("output-unsupported", output_task.clone())) + .await, + Err(RuntimeError::UnsupportedCapabilities(missing)) + if missing == vec!["feature:OutputArtifacts"] + )); + assert!(matches!( + store.load("output-task").await, + Err(RuntimeError::NotFound { .. }) + )); + + output_task.unit_id = "output-task-supported".into(); + let supported = Arc::new(TestDriver::new()); + let (client, _) = supported.client(); + let observation = client + .apply(&apply("output-supported", output_task.clone())) + .await + .unwrap(); + assert_eq!(observation.outputs.len(), 1); + assert_eq!(observation.outputs[0].name, "result"); + assert!(observation.outputs[0].size_bytes <= 16); + let mut oversized = observation; + oversized.outputs[0].size_bytes = 17; + assert!(oversized.validate_against(&output_task).is_err()); + + let mut confidential_driver = TestDriver::new(); + confidential_driver + .capabilities + .isolation_levels + .push(IsolationLevel::Confidential); + let confidential_driver = Arc::new(confidential_driver); + let (client, store) = confidential_driver.client(); + let mut confidential = service("confidential-service", 1); + confidential.isolation = IsolationLevel::Confidential; + assert!(matches!( + client + .apply(&apply("confidential-unsupported", confidential)) + .await, + Err(RuntimeError::UnsupportedCapabilities(missing)) + if missing == vec!["feature:Attestation"] + )); + assert!(matches!( + store.load("confidential-service").await, + Err(RuntimeError::NotFound { .. }) + )); + + let mut supported_confidential = TestDriver::new(); + supported_confidential + .capabilities + .isolation_levels + .push(IsolationLevel::Confidential); + supported_confidential + .capabilities + .features + .push(RuntimeFeature::Attestation); + let supported_confidential = Arc::new(supported_confidential); + let (client, _) = supported_confidential.client(); + let mut confidential = service("confidential-supported", 1); + confidential.isolation = IsolationLevel::Confidential; + let observation = client + .apply(&apply("confidential-supported", confidential)) + .await + .unwrap(); + assert!(observation.provider_attestation.is_some()); +} + +#[tokio::test] +async fn apply_stop_and_exec_postconditions_fail_closed() { + let driver = Arc::new(TestDriver::new()); + driver + .return_starting_on_apply + .store(true, Ordering::SeqCst); + let (client, _) = driver.client(); + let spec = service("postconditions", 1); + assert!(matches!( + client + .apply(&apply("postconditions-starting", spec.clone())) + .await, + Err(RuntimeError::Protocol(message)) if message.contains("invalid Service result") + )); + + driver + .return_starting_on_apply + .store(false, Ordering::SeqCst); + client + .apply(&apply("postconditions-running", spec)) + .await + .unwrap(); + driver.return_running_on_stop.store(true, Ordering::SeqCst); + let stop = action("postconditions-stop", "postconditions", 1); + assert!(matches!( + client.stop(&stop).await, + Err(RuntimeError::Protocol(message)) if message.contains("nonterminal state") + )); + + driver.return_running_on_stop.store(false, Ordering::SeqCst); + client.stop(&stop).await.unwrap(); + assert!(matches!( + client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-stopped".into(), + unit_id: "postconditions".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }) + .await, + Err(RuntimeError::InvalidRequest(message)) if message.contains("running unit") + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 0); + + let unknown_driver = Arc::new(TestDriver::new()); + unknown_driver + .return_unknown_on_apply + .store(true, Ordering::SeqCst); + let (unknown_client, _) = unknown_driver.client(); + assert!(matches!( + unknown_client + .apply(&apply( + "postconditions-unknown", + service("postconditions-unknown", 1), + )) + .await, + Err(RuntimeError::Protocol(message)) + if message.contains("without provider identity") + )); +} + +#[tokio::test] +async fn exec_result_must_bind_the_complete_unit_spec() { + let driver = Arc::new(TestDriver::new()); + let (client, _) = driver.client(); + client + .apply(&apply("exec-binding-apply", service("exec-binding", 1))) + .await + .unwrap(); + driver.substitute_exec_spec.store(true, Ordering::SeqCst); + + assert!(matches!( + client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-binding-request".into(), + unit_id: "exec-binding".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }) + .await, + Err(RuntimeError::Protocol(message)) + if message.contains("does not match the unit specification") + )); +} + +#[test] +fn top_level_log_exec_and_inspection_records_require_current_schemas() { + let mut query = RuntimeLogQuery { + schema: RuntimeLogQuery::SCHEMA.into(), + unit_id: "schema-unit".into(), + generation: 1, + cursor: None, + limit: 1, + stream: None, + }; + query.validate().unwrap(); + query.schema = "a3s.runtime.log-query.v0".into(); + assert!(query.validate().is_err()); + + let mut chunk = RuntimeLogChunk { + schema: RuntimeLogChunk::SCHEMA.into(), + cursor: "cursor".into(), + sequence: 1, + observed_at_ms: NOW, + stream: RuntimeLogStream::Stdout, + data: String::new(), + }; + chunk.validate().unwrap(); + chunk.schema.clear(); + assert!(chunk.validate().is_err()); + + let mut exec = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "schema-exec".into(), + unit_id: "schema-unit".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1, + deadline_at_ms: None, + }; + exec.validate().unwrap(); + exec.schema = "future".into(); + assert!(exec.validate().is_err()); + + let inspection = RuntimeInspection::NotFound { + schema: "future".into(), + unit_id: "schema-unit".into(), + last_generation: None, + }; + assert!(inspection.validate().is_err()); + + assert!( + serde_json::from_value::(serde_json::json!({ + "unit_id": "schema-unit", + "generation": 1, + "cursor": null, + "limit": 1, + "stream": null + })) + .is_err() + ); +} + #[tokio::test] async fn provider_identity_substitution_and_unordered_logs_are_rejected() { let driver = Arc::new(TestDriver::new()); @@ -490,6 +848,7 @@ async fn provider_identity_substitution_and_unordered_logs_are_rejected() { assert!(matches!( client .logs(&RuntimeLogQuery { + schema: RuntimeLogQuery::SCHEMA.into(), unit_id: "identity-service".into(), generation: 1, cursor: None, @@ -511,6 +870,7 @@ async fn log_and_exec_capabilities_use_the_active_generation() { .unwrap(); let logs = client .logs(&RuntimeLogQuery { + schema: RuntimeLogQuery::SCHEMA.into(), unit_id: "service-tools".into(), generation: 1, cursor: None, @@ -523,11 +883,13 @@ async fn log_and_exec_capabilities_use_the_active_generation() { let result = client .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), request_id: "exec-tools".into(), unit_id: "service-tools".into(), generation: 1, command: vec!["/bin/true".into()], timeout_ms: 1_000, + deadline_at_ms: None, }) .await .unwrap(); @@ -536,6 +898,7 @@ async fn log_and_exec_capabilities_use_the_active_generation() { assert!(matches!( client .logs(&RuntimeLogQuery { + schema: RuntimeLogQuery::SCHEMA.into(), unit_id: "service-tools".into(), generation: 2, cursor: None, @@ -574,7 +937,7 @@ async fn provider_loss_becomes_a_durable_unknown_observation() { driver.missing_on_inspect.store(true, Ordering::SeqCst); let inspection = client.inspect("service-loss").await.unwrap(); - let RuntimeInspection::Found { observation } = inspection else { + let RuntimeInspection::Found { observation, .. } = inspection else { panic!("a previously observed provider unit must not become definitively absent"); }; assert_eq!(observation.state, RuntimeUnitState::Unknown); @@ -589,6 +952,39 @@ async fn provider_loss_becomes_a_durable_unknown_observation() { ); } +#[tokio::test] +async fn same_generation_apply_recovers_a_lost_provider_unit_once() { + let driver = Arc::new(TestDriver::new()); + let (client, store) = driver.client(); + let spec = service("service-recovery", 1); + let running = client + .apply(&apply("apply-recovery", spec.clone())) + .await + .unwrap(); + + driver.missing_on_inspect.store(true, Ordering::SeqCst); + let RuntimeInspection::Found { observation, .. } = + client.inspect("service-recovery").await.unwrap() + else { + panic!("provider loss must first become a durable unknown observation"); + }; + assert_eq!(observation.state, RuntimeUnitState::Unknown); + + driver.missing_on_inspect.store(false, Ordering::SeqCst); + let recovery = apply("reapply-recovery", spec); + let recovered = client.apply(&recovery).await.unwrap(); + assert_eq!(recovered.state, RuntimeUnitState::Running); + assert_ne!(recovered.provider_resource_id, running.provider_resource_id); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 2); + + assert_eq!(client.apply(&recovery).await.unwrap(), recovered); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 2); + assert_eq!( + store.load("service-recovery").await.unwrap().observation, + recovered + ); +} + #[tokio::test] async fn terminal_observations_are_immutable() { let driver = Arc::new(TestDriver::new()); @@ -643,20 +1039,24 @@ struct CountingFactory { client: Arc, } +#[async_trait] impl RuntimeProviderFactory for CountingFactory { fn provider_id(&self) -> &ProviderId { &self.provider } - fn create(&self) -> RuntimeResult> { + async fn create(&self) -> RuntimeResult> { self.creates.fetch_add(1, Ordering::SeqCst); Ok(self.client.clone()) } } -#[test] -fn provider_registry_never_falls_back_or_replaces_a_factory() { - let driver = Arc::new(TestDriver::new()); +#[tokio::test] +async fn provider_registry_never_falls_back_or_replaces_a_factory() { + let mut driver = TestDriver::new(); + driver.provider = ProviderId::parse("explicit-provider").unwrap(); + driver.capabilities.provider_id = driver.provider.clone(); + let driver = Arc::new(driver); let (client, _) = driver.client(); let factory = Arc::new(CountingFactory { provider: ProviderId::parse("explicit-provider").unwrap(), @@ -668,14 +1068,33 @@ fn provider_registry_never_falls_back_or_replaces_a_factory() { let missing = ProviderId::parse("missing-provider").unwrap(); assert!(matches!( - registry.connect(&missing), + registry.connect(&missing).await, Err(RuntimeError::ProviderUnavailable(_)) )); assert_eq!(factory.creates.load(Ordering::SeqCst), 0); assert!(registry.register(factory.clone()).is_err()); assert_eq!(factory.creates.load(Ordering::SeqCst), 0); - registry.connect(factory.provider_id()).unwrap(); + registry.connect(factory.provider_id()).await.unwrap(); + assert_eq!(factory.creates.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn provider_registry_rejects_a_factory_client_identity_mismatch() { + let driver = Arc::new(TestDriver::new()); + let (client, _) = driver.client(); + let factory = Arc::new(CountingFactory { + provider: ProviderId::parse("registered-provider").unwrap(), + creates: AtomicUsize::new(0), + client: Arc::new(client), + }); + let mut registry = RuntimeClientRegistry::new(); + registry.register(factory.clone()).unwrap(); + + assert!(matches!( + registry.connect(factory.provider_id()).await, + Err(RuntimeError::Protocol(message)) if message.contains("created client reporting") + )); assert_eq!(factory.creates.load(Ordering::SeqCst), 1); } From 79b147dddb61b9c83f14341820349638723793da Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 12:02:38 +0800 Subject: [PATCH 02/32] test: select the intended state lock fixture --- tests/general_runtime.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 6cfc4ad..5c277fb 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -1101,6 +1101,7 @@ async fn provider_registry_rejects_a_factory_client_identity_mismatch() { #[cfg(unix)] #[tokio::test] async fn file_state_store_rejects_symbolic_link_boundaries() { + use sha2::{Digest, Sha256}; use std::os::unix::fs::symlink; let directory = tempfile::tempdir().unwrap(); @@ -1124,12 +1125,9 @@ async fn file_state_store_rejects_symbolic_link_boundaries() { .reserve_apply(&apply("secure", task("secure-unit", 1)), NOW) .await .unwrap(); - let lock_path = std::fs::read_dir(root.join("locks")) - .unwrap() - .next() - .unwrap() - .unwrap() - .path(); + let lock_path = root + .join("locks") + .join(format!("{:x}.lock", Sha256::digest(b"secure-unit"))); std::fs::remove_file(&lock_path).unwrap(); let lock_target = root.join("lock-target"); std::fs::write(&lock_target, b"do not follow").unwrap(); From 661703eeadc402fb91549ebe52483a3fa594a4ea Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 12:14:35 +0800 Subject: [PATCH 03/32] feat: bound operations with cross-process leases --- Cargo.toml | 2 +- src/lib.rs | 5 +- src/managed.rs | 141 ++++++++++++++++-- src/state/file.rs | 48 +++++- src/state/mod.rs | 9 ++ tests/general_runtime.rs | 314 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 499 insertions(+), 20 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 367e252..28c4f87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ serde_json = "1" sha2 = "0.10" tempfile = "3" thiserror = "2" -tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/src/lib.rs b/src/lib.rs index 85f3ab8..85eb1dd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,7 @@ pub use managed::ManagedRuntimeClient; pub use provider::ProviderId; pub use registry::{RuntimeClientRegistry, RuntimeProviderFactory}; pub use state::{ - FileRuntimeStateStore, RuntimeActionKind, RuntimeRequestKind, RuntimeRequestReceipt, - RuntimeRequestState, RuntimeStateReservation, RuntimeStateStore, RuntimeUnitRecord, + FileRuntimeStateStore, RuntimeActionKind, RuntimeOperationLease, RuntimeRequestKind, + RuntimeRequestReceipt, RuntimeRequestState, RuntimeStateReservation, RuntimeStateStore, + RuntimeUnitRecord, }; diff --git a/src/managed.rs b/src/managed.rs index 6612e62..7fd8b84 100644 --- a/src/managed.rs +++ b/src/managed.rs @@ -8,7 +8,9 @@ use crate::{ RuntimeStateStore, SystemRuntimeClock, }; use async_trait::async_trait; +use std::future::Future; use std::sync::Arc; +use std::time::Duration; /// Shared durable lifecycle implementation used by provider integrations. pub struct ManagedRuntimeClient { @@ -55,6 +57,44 @@ impl ManagedRuntimeClient { } Ok(()) } + + async fn bounded( + &self, + deadline_at_ms: Option, + stage: &'static str, + future: F, + ) -> RuntimeResult + where + T: Send, + F: Future> + Send, + { + let Some(deadline_at_ms) = deadline_at_ms else { + return future.await; + }; + let now_ms = self.clock.now_ms(); + let Some(remaining_ms) = deadline_at_ms.checked_sub(now_ms) else { + return Err(RuntimeError::DeadlineExceeded(format!( + "request expired before {stage}" + ))); + }; + if remaining_ms == 0 { + return Err(RuntimeError::DeadlineExceeded(format!( + "request expired before {stage}" + ))); + } + tokio::time::timeout(Duration::from_millis(remaining_ms), future) + .await + .map_err(|_| { + RuntimeError::DeadlineExceeded(format!("request deadline elapsed during {stage}")) + })? + } + + fn exec_deadline(&self, request: &RuntimeExecRequest) -> u64 { + let relative = self.clock.now_ms().saturating_add(request.timeout_ms); + request + .deadline_at_ms + .map_or(relative, |absolute| absolute.min(relative)) + } } #[async_trait] @@ -66,7 +106,13 @@ impl RuntimeClient for ManagedRuntimeClient { async fn apply(&self, request: &RuntimeApplyRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; self.check_deadline(request.deadline_at_ms)?; - let capabilities = self.checked_capabilities().await?; + let capabilities = self + .bounded( + request.deadline_at_ms, + "capability query", + self.checked_capabilities(), + ) + .await?; let missing = capabilities .missing_for(&request.spec) .map_err(RuntimeError::InvalidRequest)?; @@ -74,6 +120,15 @@ impl RuntimeClient for ManagedRuntimeClient { return Err(RuntimeError::UnsupportedCapabilities(missing)); } + let _lease = self + .bounded( + request.deadline_at_ms, + "operation lease wait", + self.state.acquire_operation_lease(&request.spec.unit_id), + ) + .await?; + self.check_deadline(request.deadline_at_ms)?; + let reservation = self .state .reserve_apply(request, self.clock.now_ms()) @@ -85,8 +140,12 @@ impl RuntimeClient for ManagedRuntimeClient { } let observation = self - .driver - .apply(&request.spec, &reservation.record.observation) + .bounded( + request.deadline_at_ms, + "provider apply", + self.driver + .apply(&request.spec, &reservation.record.observation), + ) .await?; observation .validate_against(&request.spec) @@ -100,6 +159,7 @@ impl RuntimeClient for ManagedRuntimeClient { } async fn inspect(&self, unit_id: &str) -> RuntimeResult { + let _lease = self.state.acquire_operation_lease(unit_id).await?; let record = match self.state.load(unit_id).await { Ok(record) => record, Err(RuntimeError::NotFound { .. }) => { @@ -161,12 +221,26 @@ impl RuntimeClient for ManagedRuntimeClient { async fn stop(&self, request: &RuntimeActionRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; self.check_deadline(request.deadline_at_ms)?; - let capabilities = self.checked_capabilities().await?; + let capabilities = self + .bounded( + request.deadline_at_ms, + "capability query", + self.checked_capabilities(), + ) + .await?; if !capabilities.supports_feature(crate::contract::RuntimeFeature::Stop) { return Err(RuntimeError::UnsupportedCapabilities(vec![ "feature:Stop".into() ])); } + let _lease = self + .bounded( + request.deadline_at_ms, + "operation lease wait", + self.state.acquire_operation_lease(&request.unit_id), + ) + .await?; + self.check_deadline(request.deadline_at_ms)?; let reservation = self .state .reserve_action(RuntimeActionKind::Stop, request, self.clock.now_ms()) @@ -179,7 +253,13 @@ impl RuntimeClient for ManagedRuntimeClient { })?), }); } - let observation = self.driver.stop(&reservation.record, request).await?; + let observation = self + .bounded( + request.deadline_at_ms, + "provider stop", + self.driver.stop(&reservation.record, request), + ) + .await?; observation .validate_against(&reservation.record.spec) .map_err(RuntimeError::Protocol)?; @@ -197,12 +277,26 @@ impl RuntimeClient for ManagedRuntimeClient { async fn remove(&self, request: &RuntimeActionRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; self.check_deadline(request.deadline_at_ms)?; - let capabilities = self.checked_capabilities().await?; + let capabilities = self + .bounded( + request.deadline_at_ms, + "capability query", + self.checked_capabilities(), + ) + .await?; if !capabilities.supports_feature(crate::contract::RuntimeFeature::Remove) { return Err(RuntimeError::UnsupportedCapabilities(vec![ "feature:Remove".into(), ])); } + let _lease = self + .bounded( + request.deadline_at_ms, + "operation lease wait", + self.state.acquire_operation_lease(&request.unit_id), + ) + .await?; + self.check_deadline(request.deadline_at_ms)?; let reservation = self .state .reserve_action(RuntimeActionKind::Remove, request, self.clock.now_ms()) @@ -212,7 +306,13 @@ impl RuntimeClient for ManagedRuntimeClient { RuntimeError::Protocol("completed remove receipt has no removal".into()) }); } - let removal = self.driver.remove(&reservation.record, request).await?; + let removal = self + .bounded( + request.deadline_at_ms, + "provider remove", + self.driver.remove(&reservation.record, request), + ) + .await?; removal.validate().map_err(RuntimeError::Protocol)?; if removal.request_id != request.request_id || removal.unit_id != request.unit_id @@ -234,6 +334,7 @@ impl RuntimeClient for ManagedRuntimeClient { "feature:Logs".into() ])); } + let _lease = self.state.acquire_operation_lease(&query.unit_id).await?; let record = self.state.load(&query.unit_id).await?; ensure_current_generation(&record, query.generation)?; let chunks = self.driver.logs(&record, query).await?; @@ -253,12 +354,28 @@ impl RuntimeClient for ManagedRuntimeClient { async fn exec(&self, request: &RuntimeExecRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; - let capabilities = self.checked_capabilities().await?; + let deadline_at_ms = Some(self.exec_deadline(request)); + self.check_deadline(deadline_at_ms)?; + let capabilities = self + .bounded( + deadline_at_ms, + "capability query", + self.checked_capabilities(), + ) + .await?; if !capabilities.supports_feature(crate::contract::RuntimeFeature::Exec) { return Err(RuntimeError::UnsupportedCapabilities(vec![ "feature:Exec".into() ])); } + let _lease = self + .bounded( + deadline_at_ms, + "operation lease wait", + self.state.acquire_operation_lease(&request.unit_id), + ) + .await?; + self.check_deadline(deadline_at_ms)?; let record = self.state.load(&request.unit_id).await?; ensure_current_generation(&record, request.generation)?; if record.observation.state != RuntimeUnitState::Running { @@ -267,7 +384,13 @@ impl RuntimeClient for ManagedRuntimeClient { request.unit_id, record.observation.state ))); } - let result = self.driver.exec(&record, request).await?; + let result = self + .bounded( + deadline_at_ms, + "provider exec", + self.driver.exec(&record, request), + ) + .await?; result.validate().map_err(RuntimeError::Protocol)?; result .observation diff --git a/src/state/file.rs b/src/state/file.rs index 5523a63..7bb8284 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -1,6 +1,6 @@ use super::{ - RuntimeActionKind, RuntimeRequestKind, RuntimeRequestReceipt, RuntimeRequestState, - RuntimeStateReservation, RuntimeStateStore, RuntimeUnitRecord, + RuntimeActionKind, RuntimeOperationLease, RuntimeRequestKind, RuntimeRequestReceipt, + RuntimeRequestState, RuntimeStateReservation, RuntimeStateStore, RuntimeUnitRecord, }; use crate::contract::{ RuntimeActionRequest, RuntimeApplyRequest, RuntimeObservation, RuntimeRemoval, RuntimeUnitState, @@ -23,6 +23,18 @@ impl FileRuntimeStateStore { Self { root: root.into() } } + fn acquire_operation_lease_sync(&self, unit_id: &str) -> RuntimeResult { + validate_unit_id(unit_id)?; + ensure_directory(&self.root)?; + let operations = self.root.join("operations"); + ensure_directory(&operations)?; + let path = operations.join(format!("{}.lock", storage_key(unit_id))); + let file = owner_only_open(&path, "Runtime operation lease")?; + file.lock_exclusive() + .map_err(io_error("lock Runtime operation"))?; + Ok(FileOperationLease(file)) + } + fn reserve_apply_sync( &self, request: RuntimeApplyRequest, @@ -246,7 +258,10 @@ impl FileRuntimeStateStore { ensure_directory(&self.root)?; let locks = self.root.join("locks"); ensure_directory(&locks)?; - let file = owner_only_open(&locks.join(format!("{}.lock", storage_key(unit_id))))?; + let file = owner_only_open( + &locks.join(format!("{}.lock", storage_key(unit_id))), + "Runtime state lock", + )?; file.lock_exclusive() .map_err(io_error("lock Runtime unit"))?; Ok(StateLock(file)) @@ -262,6 +277,19 @@ impl FileRuntimeStateStore { #[async_trait] impl RuntimeStateStore for FileRuntimeStateStore { + async fn acquire_operation_lease( + &self, + unit_id: &str, + ) -> RuntimeResult> { + let store = self.clone(); + let unit_id = unit_id.to_owned(); + let lease = + tokio::task::spawn_blocking(move || store.acquire_operation_lease_sync(&unit_id)) + .await + .map_err(task_error)??; + Ok(Box::new(lease)) + } + async fn reserve_apply( &self, request: &RuntimeApplyRequest, @@ -426,6 +454,16 @@ impl Drop for StateLock { } } +struct FileOperationLease(File); + +impl RuntimeOperationLease for FileOperationLease {} + +impl Drop for FileOperationLease { + fn drop(&mut self) { + let _ = FileExt::unlock(&self.0); + } +} + fn storage_key(unit_id: &str) -> String { format!("{:x}", Sha256::digest(unit_id.as_bytes())) } @@ -455,8 +493,8 @@ fn ensure_directory(path: &Path) -> RuntimeResult<()> { Ok(()) } -fn owner_only_open(path: &Path) -> RuntimeResult { - reject_symlink(path, "Runtime state lock")?; +fn owner_only_open(path: &Path, label: &str) -> RuntimeResult { + reject_symlink(path, label)?; let mut options = OpenOptions::new(); options.create(true).read(true).write(true); #[cfg(unix)] diff --git a/src/state/mod.rs b/src/state/mod.rs index c269adf..f4bd29d 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -13,8 +13,17 @@ pub use record::{ RuntimeStateReservation, RuntimeUnitRecord, }; +/// Owned guard for one unit's cross-process operation lease. Implementations +/// release the lease when the guard is dropped. +pub trait RuntimeOperationLease: Send {} + #[async_trait] pub trait RuntimeStateStore: Send + Sync { + async fn acquire_operation_lease( + &self, + unit_id: &str, + ) -> RuntimeResult>; + async fn reserve_apply( &self, request: &RuntimeApplyRequest, diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 5c277fb..9df782c 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -9,13 +9,16 @@ use a3s_runtime::contract::{ }; use a3s_runtime::{ verify_runtime_provider, FileRuntimeStateStore, ManagedRuntimeClient, ProviderId, - RuntimeClient, RuntimeClientRegistry, RuntimeClock, RuntimeConformanceCase, RuntimeDriver, - RuntimeError, RuntimeProviderFactory, RuntimeResult, RuntimeStateStore, RuntimeUnitRecord, + RuntimeActionKind, RuntimeClient, RuntimeClientRegistry, RuntimeClock, RuntimeConformanceCase, + RuntimeDriver, RuntimeError, RuntimeOperationLease, RuntimeProviderFactory, RuntimeResult, + RuntimeStateReservation, RuntimeStateStore, RuntimeUnitRecord, }; use async_trait::async_trait; use std::collections::BTreeMap; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Notify; const NOW: u64 = 1_000; const IMAGE_MEDIA_TYPE: &str = "application/vnd.oci.image.manifest.v1+json"; @@ -29,6 +32,25 @@ impl RuntimeClock for FixedClock { } } +#[derive(Debug)] +struct ManualClock(AtomicU64); + +impl ManualClock { + fn new(now_ms: u64) -> Self { + Self(AtomicU64::new(now_ms)) + } + + fn set(&self, now_ms: u64) { + self.0.store(now_ms, Ordering::SeqCst); + } +} + +impl RuntimeClock for ManualClock { + fn now_ms(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } +} + struct TestDriver { provider: ProviderId, capabilities: RuntimeCapabilities, @@ -45,6 +67,9 @@ struct TestDriver { return_unknown_on_apply: AtomicBool, return_running_on_stop: AtomicBool, substitute_exec_spec: AtomicBool, + hang_capabilities: AtomicBool, + hang_apply: AtomicBool, + hang_exec: AtomicBool, } impl TestDriver { @@ -66,6 +91,9 @@ impl TestDriver { return_unknown_on_apply: AtomicBool::new(false), return_running_on_stop: AtomicBool::new(false), substitute_exec_spec: AtomicBool::new(false), + hang_capabilities: AtomicBool::new(false), + hang_apply: AtomicBool::new(false), + hang_exec: AtomicBool::new(false), } } @@ -79,6 +107,55 @@ impl TestDriver { } } +struct SignalingStateStore { + inner: Arc, + lease_started: Arc, +} + +#[async_trait] +impl RuntimeStateStore for SignalingStateStore { + async fn acquire_operation_lease( + &self, + unit_id: &str, + ) -> RuntimeResult> { + self.lease_started.notify_one(); + self.inner.acquire_operation_lease(unit_id).await + } + + async fn reserve_apply( + &self, + request: &RuntimeApplyRequest, + now_ms: u64, + ) -> RuntimeResult { + self.inner.reserve_apply(request, now_ms).await + } + + async fn reserve_action( + &self, + kind: RuntimeActionKind, + request: &RuntimeActionRequest, + now_ms: u64, + ) -> RuntimeResult { + self.inner.reserve_action(kind, request, now_ms).await + } + + async fn load(&self, unit_id: &str) -> RuntimeResult { + self.inner.load(unit_id).await + } + + async fn update_observation( + &self, + request_id: Option<&str>, + observation: &RuntimeObservation, + ) -> RuntimeResult { + self.inner.update_observation(request_id, observation).await + } + + async fn complete_removal(&self, removal: &RuntimeRemoval) -> RuntimeResult { + self.inner.complete_removal(removal).await + } +} + #[async_trait] impl RuntimeDriver for TestDriver { fn provider_id(&self) -> &ProviderId { @@ -86,6 +163,9 @@ impl RuntimeDriver for TestDriver { } async fn capabilities(&self) -> RuntimeResult { + if self.hang_capabilities.load(Ordering::SeqCst) { + return std::future::pending::>().await; + } Ok(self.capabilities.clone()) } @@ -95,6 +175,9 @@ impl RuntimeDriver for TestDriver { current: &RuntimeObservation, ) -> RuntimeResult { self.apply_calls.fetch_add(1, Ordering::SeqCst); + if self.hang_apply.load(Ordering::SeqCst) { + return std::future::pending::>().await; + } if self.fail_next_apply.swap(false, Ordering::SeqCst) { return Err(RuntimeError::Transport("ambiguous apply".into())); } @@ -244,6 +327,9 @@ impl RuntimeDriver for TestDriver { request: &RuntimeExecRequest, ) -> RuntimeResult { self.exec_calls.fetch_add(1, Ordering::SeqCst); + if self.hang_exec.load(Ordering::SeqCst) { + return std::future::pending::>().await; + } let mut observation = unit.observation.clone(); if self.substitute_exec_spec.load(Ordering::SeqCst) { observation.spec_digest = digest('f'); @@ -926,6 +1012,228 @@ async fn expired_requests_never_reserve_or_dispatch() { )); } +#[tokio::test] +async fn file_operation_leases_serialize_one_unit_but_not_different_units() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let first = store.acquire_operation_lease("lease-unit").await.unwrap(); + + assert!(tokio::time::timeout( + Duration::from_millis(20), + store.acquire_operation_lease("lease-unit"), + ) + .await + .is_err()); + let different = tokio::time::timeout( + Duration::from_millis(100), + store.acquire_operation_lease("different-unit"), + ) + .await + .expect("a different unit must not wait behind the held lease") + .unwrap(); + + drop(different); + drop(first); +} + +#[tokio::test] +async fn subprocess_operation_lease_helper() { + let Ok(root) = std::env::var("A3S_RUNTIME_LEASE_HELPER_ROOT") else { + return; + }; + let unit_id = std::env::var("A3S_RUNTIME_LEASE_HELPER_UNIT").unwrap(); + let ready = std::env::var("A3S_RUNTIME_LEASE_HELPER_READY").unwrap(); + let release = std::env::var("A3S_RUNTIME_LEASE_HELPER_RELEASE").unwrap(); + let store = FileRuntimeStateStore::new(root); + let _lease = store.acquire_operation_lease(&unit_id).await.unwrap(); + std::fs::write(&ready, b"ready").unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + while !std::path::Path::new(&release).is_file() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("parent did not release the subprocess lease"); +} + +#[tokio::test] +async fn file_operation_lease_serializes_independent_processes() { + use std::process::Command; + + let directory = tempfile::tempdir().unwrap(); + let ready = directory.path().join("helper.ready"); + let release = directory.path().join("helper.release"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("subprocess_operation_lease_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env("A3S_RUNTIME_LEASE_HELPER_ROOT", directory.path()) + .env("A3S_RUNTIME_LEASE_HELPER_UNIT", "cross-process-unit") + .env("A3S_RUNTIME_LEASE_HELPER_READY", &ready) + .env("A3S_RUNTIME_LEASE_HELPER_RELEASE", &release) + .spawn() + .unwrap(); + + tokio::time::timeout(Duration::from_secs(10), async { + while !ready.is_file() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("subprocess did not acquire the operation lease"); + + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let waiting_store = store.clone(); + let waiter = tokio::spawn(async move { + waiting_store + .acquire_operation_lease("cross-process-unit") + .await + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!waiter.is_finished()); + let different = tokio::time::timeout( + Duration::from_millis(100), + store.acquire_operation_lease("cross-process-other"), + ) + .await + .expect("different-unit cross-process lease was blocked") + .unwrap(); + drop(different); + + std::fs::write(&release, b"release").unwrap(); + let acquired = tokio::time::timeout(Duration::from_secs(10), waiter) + .await + .expect("same-unit waiter did not acquire after subprocess release") + .unwrap() + .unwrap(); + drop(acquired); + assert!(child.wait().unwrap().success()); +} + +#[tokio::test] +async fn deadline_is_rechecked_after_the_operation_lease_wait() { + let directory = tempfile::tempdir().unwrap(); + let file_store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let held = file_store + .acquire_operation_lease("post-lease-deadline") + .await + .unwrap(); + let lease_started = Arc::new(Notify::new()); + let state: Arc = Arc::new(SignalingStateStore { + inner: file_store.clone(), + lease_started: lease_started.clone(), + }); + let driver = Arc::new(TestDriver::new()); + let clock = Arc::new(ManualClock::new(NOW)); + let client = ManagedRuntimeClient::with_clock(state, driver.clone(), clock.clone()); + let mut request = apply("post-lease-deadline", service("post-lease-deadline", 1)); + request.deadline_at_ms = Some(NOW + 1_000); + + let operation = tokio::spawn(async move { client.apply(&request).await }); + lease_started.notified().await; + clock.set(NOW + 1_000); + drop(held); + + assert!(matches!( + operation.await.unwrap(), + Err(RuntimeError::DeadlineExceeded(message)) + if message.contains("before provider dispatch") + )); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 0); + assert!(matches!( + file_store.load("post-lease-deadline").await, + Err(RuntimeError::NotFound { .. }) + )); +} + +#[tokio::test] +async fn deadlines_bound_capability_queries_and_provider_dispatch() { + let capability_driver = Arc::new(TestDriver::new()); + capability_driver + .hang_capabilities + .store(true, Ordering::SeqCst); + let (client, store) = capability_driver.client(); + let mut request = apply("capability-timeout", service("capability-timeout", 1)); + request.deadline_at_ms = Some(NOW + 10); + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(message)) + if message.contains("capability query") + )); + assert!(matches!( + store.load("capability-timeout").await, + Err(RuntimeError::NotFound { .. }) + )); + assert_eq!(capability_driver.apply_calls.load(Ordering::SeqCst), 0); + + let provider_driver = Arc::new(TestDriver::new()); + provider_driver.hang_apply.store(true, Ordering::SeqCst); + let (client, store) = provider_driver.client(); + let mut request = apply("provider-timeout", service("provider-timeout", 1)); + request.deadline_at_ms = Some(NOW + 10); + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider apply") + )); + let pending = store.load("provider-timeout").await.unwrap(); + assert_eq!( + pending.receipt("provider-timeout").unwrap().state, + a3s_runtime::RuntimeRequestState::Pending + ); + assert_eq!(provider_driver.apply_calls.load(Ordering::SeqCst), 1); + + provider_driver.hang_apply.store(false, Ordering::SeqCst); + assert_eq!( + client.apply(&request).await.unwrap().state, + RuntimeUnitState::Running + ); + assert_eq!(provider_driver.apply_calls.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn exec_uses_the_smaller_relative_or_absolute_deadline() { + let driver = Arc::new(TestDriver::new()); + let (client, _) = driver.client(); + client + .apply(&apply("exec-deadline-apply", service("exec-deadline", 1))) + .await + .unwrap(); + driver.hang_exec.store(true, Ordering::SeqCst); + + assert!(matches!( + client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-relative-deadline".into(), + unit_id: "exec-deadline".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 10, + deadline_at_ms: Some(NOW + 1_000), + }) + .await, + Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider exec") + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + + assert!(matches!( + client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-absolute-deadline".into(), + unit_id: "exec-deadline".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: Some(NOW), + }) + .await, + Err(RuntimeError::DeadlineExceeded(_)) + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); +} + #[tokio::test] async fn provider_loss_becomes_a_durable_unknown_observation() { let driver = Arc::new(TestDriver::new()); From 4327c60037856e284f856663884f4823bb4638b1 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 12:28:34 +0800 Subject: [PATCH 04/32] feat: add durable request journal and exec replay --- src/error.rs | 2 + src/managed.rs | 20 +- src/state/file.rs | 674 ++++++++++++++++++++++++++++++++------- src/state/mod.rs | 17 +- src/state/record.rs | 190 +++++++---- tests/general_runtime.rs | 235 +++++++++++++- 6 files changed, 932 insertions(+), 206 deletions(-) diff --git a/src/error.rs b/src/error.rs index a97c042..aab5d64 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,6 +10,8 @@ pub enum RuntimeError { NotFound { unit_id: String }, #[error("Runtime request {request_id:?} conflicts with a prior request")] RequestConflict { request_id: String }, + #[error("Runtime request {request_id:?} for unit {unit_id:?} was not found")] + RequestNotFound { unit_id: String, request_id: String }, #[error( "Runtime unit {unit_id:?} rejected stale generation {requested}; current generation is {current}" )] diff --git a/src/managed.rs b/src/managed.rs index 7fd8b84..2e948c3 100644 --- a/src/managed.rs +++ b/src/managed.rs @@ -376,25 +376,26 @@ impl RuntimeClient for ManagedRuntimeClient { ) .await?; self.check_deadline(deadline_at_ms)?; - let record = self.state.load(&request.unit_id).await?; - ensure_current_generation(&record, request.generation)?; - if record.observation.state != RuntimeUnitState::Running { - return Err(RuntimeError::InvalidRequest(format!( - "Runtime exec requires a running unit; {:?} is {:?}", - request.unit_id, record.observation.state - ))); + let reservation = self + .state + .reserve_exec(request, self.clock.now_ms()) + .await?; + if !reservation.dispatch { + return reservation.receipt.exec_result.ok_or_else(|| { + RuntimeError::Protocol("completed exec receipt has no result".into()) + }); } let result = self .bounded( deadline_at_ms, "provider exec", - self.driver.exec(&record, request), + self.driver.exec(&reservation.record, request), ) .await?; result.validate().map_err(RuntimeError::Protocol)?; result .observation - .validate_against(&record.spec) + .validate_against(&reservation.record.spec) .map_err(RuntimeError::Protocol)?; if result.request_id != request.request_id || result.observation.unit_id != request.unit_id @@ -404,6 +405,7 @@ impl RuntimeClient for ManagedRuntimeClient { "provider exec changed immutable request identity".into(), )); } + self.state.complete_exec(&result).await?; Ok(result) } } diff --git a/src/state/file.rs b/src/state/file.rs index 7bb8284..fb79ff6 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -3,16 +3,22 @@ use super::{ RuntimeRequestState, RuntimeStateReservation, RuntimeStateStore, RuntimeUnitRecord, }; use crate::contract::{ - RuntimeActionRequest, RuntimeApplyRequest, RuntimeObservation, RuntimeRemoval, RuntimeUnitState, + RuntimeActionRequest, RuntimeApplyRequest, RuntimeExecRequest, RuntimeExecResult, + RuntimeObservation, RuntimeRemoval, RuntimeUnitState, }; use crate::{RuntimeError, RuntimeResult}; use async_trait::async_trait; use fs2::FileExt; +use serde::de::DeserializeOwned; +use serde::Serialize; use sha2::{Digest, Sha256}; use std::fs::{File, OpenOptions}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; +const MAX_RECORD_BYTES: u64 = 8 * 1024 * 1024; +const MAX_RECEIPT_BYTES: u64 = 40 * 1024 * 1024; + #[derive(Debug, Clone)] pub struct FileRuntimeStateStore { root: PathBuf, @@ -42,20 +48,40 @@ impl FileRuntimeStateStore { ) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; let _lock = self.lock(&request.spec.unit_id)?; - let path = self.record_path(&request.spec.unit_id)?; - let mut record = if path.exists() { - read_record(&path)? - } else { - let record = - RuntimeUnitRecord::new(&request, now_ms).map_err(RuntimeError::Protocol)?; - atomic_write(&path, &record)?; - return reservation(record, &request.request_id); + let record_path = self.record_path(&request.spec.unit_id, true)?; + let mut receipt_path = + self.request_path(&request.spec.unit_id, &request.request_id, false)?; + let existing = read_optional_receipt(&receipt_path)?; + if let Some(receipt) = &existing { + let digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + ensure_same_request(receipt, RuntimeRequestKind::Apply, &digest)?; + ensure_receipt_target(receipt, &request.spec.unit_id, request.spec.generation)?; + } + + let stored_record = read_optional_record(&record_path)?; + if stored_record.is_none() + && existing + .as_ref() + .is_some_and(|receipt| receipt.state == RuntimeRequestState::Completed) + { + return Err(RuntimeError::Protocol( + "completed Runtime apply receipt has no unit record".into(), + )); + } + let mut record_changed = stored_record.is_none(); + let mut record = match stored_record { + Some(record) => record, + None => RuntimeUnitRecord::new(&request, now_ms).map_err(RuntimeError::Protocol)?, }; - if let Some(existing) = record.receipt(&request.request_id) { - let digest = request.digest().map_err(RuntimeError::InvalidRequest)?; - ensure_same_request(existing, RuntimeRequestKind::Apply, &digest)?; - return reservation(record, &request.request_id); + if let Some(receipt) = existing + .as_ref() + .filter(|receipt| receipt.state == RuntimeRequestState::Completed) + { + if reconcile_completed_observation(&mut record, receipt)? { + atomic_write(&record_path, &record, "state record")?; + } + return Ok(reservation(record, receipt.clone())); } let current_generation = record.spec.generation; @@ -67,6 +93,11 @@ impl FileRuntimeStateStore { }); } + let receipt_is_new = existing.is_none(); + let mut receipt = existing.unwrap_or( + RuntimeRequestReceipt::pending_apply(&request).map_err(RuntimeError::Protocol)?, + ); + if request.spec.generation == current_generation { if request .spec @@ -80,28 +111,33 @@ impl FileRuntimeStateStore { generation: request.spec.generation, }); } - let mut receipt = - RuntimeRequestReceipt::pending_apply(&request).map_err(RuntimeError::Protocol)?; if !matches!( record.observation.state, RuntimeUnitState::Accepted | RuntimeUnitState::Unknown ) { receipt.complete_with_observation(record.observation.clone()); + receipt.validate().map_err(RuntimeError::Protocol)?; } - record.requests.insert(request.request_id.clone(), receipt); } else { record.spec = request.spec.clone(); record.observation = RuntimeObservation::accepted(&request.spec, now_ms) .map_err(RuntimeError::Protocol)?; record.removed_at_ms = None; - record.requests.insert( - request.request_id.clone(), - RuntimeRequestReceipt::pending_apply(&request).map_err(RuntimeError::Protocol)?, - ); + record_changed = true; + } + + if receipt_is_new || receipt.state == RuntimeRequestState::Completed { + if receipt_is_new { + receipt_path = + self.request_path(&request.spec.unit_id, &request.request_id, true)?; + } + atomic_write(&receipt_path, &receipt, "request receipt")?; } record.validate().map_err(RuntimeError::Protocol)?; - atomic_write(&path, &record)?; - reservation(record, &request.request_id) + if record_changed { + atomic_write(&record_path, &record, "state record")?; + } + Ok(reservation(record, receipt)) } fn reserve_action_sync( @@ -112,32 +148,38 @@ impl FileRuntimeStateStore { ) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; let _lock = self.lock(&request.unit_id)?; - let path = self.record_path(&request.unit_id)?; - if !path.is_file() { - return Err(RuntimeError::NotFound { - unit_id: request.unit_id, - }); - } - let mut record = read_record(&path)?; - if let Some(existing) = record.receipt(&request.request_id) { + let record_path = self.record_path(&request.unit_id, false)?; + let mut record = read_required_record(&record_path, &request.unit_id)?; + let mut receipt_path = self.request_path(&request.unit_id, &request.request_id, false)?; + if let Some(receipt) = read_optional_receipt(&receipt_path)? { let digest = request.digest().map_err(RuntimeError::InvalidRequest)?; - ensure_same_request(existing, kind.into(), &digest)?; - return reservation(record, &request.request_id); - } - if request.generation < record.spec.generation { - return Err(RuntimeError::StaleGeneration { - unit_id: request.unit_id, - requested: request.generation, - current: record.spec.generation, - }); - } - if request.generation != record.spec.generation { - return Err(RuntimeError::GenerationConflict { - unit_id: request.unit_id, - generation: request.generation, - }); + ensure_same_request(&receipt, kind.into(), &digest)?; + ensure_receipt_target(&receipt, &request.unit_id, request.generation)?; + if receipt.state == RuntimeRequestState::Completed { + let changed = match receipt.kind { + RuntimeRequestKind::Stop => { + reconcile_completed_observation(&mut record, &receipt)? + } + RuntimeRequestKind::Remove => { + reconcile_completed_removal(&mut record, &receipt)? + } + _ => false, + }; + if changed { + atomic_write(&record_path, &record, "state record")?; + } + } else { + ensure_current_generation(&record, &request.unit_id, request.generation)?; + if record.removed_at_ms.is_some() { + return Err(RuntimeError::NotFound { + unit_id: request.unit_id, + }); + } + } + return Ok(reservation(record, receipt)); } + ensure_current_generation(&record, &request.unit_id, request.generation)?; let mut receipt = RuntimeRequestReceipt::pending_action(kind, &request) .map_err(RuntimeError::Protocol)?; match kind { @@ -164,26 +206,87 @@ impl FileRuntimeStateStore { } } } - record.requests.insert(request.request_id.clone(), receipt); - record.validate().map_err(RuntimeError::Protocol)?; - atomic_write(&path, &record)?; - reservation(record, &request.request_id) + receipt.validate().map_err(RuntimeError::Protocol)?; + receipt_path = self.request_path(&request.unit_id, &request.request_id, true)?; + atomic_write(&receipt_path, &receipt, "request receipt")?; + Ok(reservation(record, receipt)) + } + + fn reserve_exec_sync( + &self, + request: RuntimeExecRequest, + ) -> RuntimeResult { + request.validate().map_err(RuntimeError::InvalidRequest)?; + let _lock = self.lock(&request.unit_id)?; + let record_path = self.record_path(&request.unit_id, false)?; + let mut record = read_required_record(&record_path, &request.unit_id)?; + let mut receipt_path = self.request_path(&request.unit_id, &request.request_id, false)?; + if let Some(receipt) = read_optional_receipt(&receipt_path)? { + let digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + ensure_same_request(&receipt, RuntimeRequestKind::Exec, &digest)?; + ensure_receipt_target(&receipt, &request.unit_id, request.generation)?; + if receipt.state == RuntimeRequestState::Completed + && reconcile_completed_exec(&mut record, &receipt)? + { + atomic_write(&record_path, &record, "state record")?; + } else if receipt.state == RuntimeRequestState::Pending { + ensure_current_generation(&record, &request.unit_id, request.generation)?; + if record.removed_at_ms.is_some() { + return Err(RuntimeError::NotFound { + unit_id: request.unit_id, + }); + } + } + return Ok(reservation(record, receipt)); + } + + ensure_current_generation(&record, &request.unit_id, request.generation)?; + if record.removed_at_ms.is_some() { + return Err(RuntimeError::NotFound { + unit_id: request.unit_id, + }); + } + if record.observation.state != RuntimeUnitState::Running { + return Err(RuntimeError::InvalidRequest(format!( + "Runtime exec requires a running unit; {:?} is {:?}", + request.unit_id, record.observation.state + ))); + } + let receipt = + RuntimeRequestReceipt::pending_exec(&request).map_err(RuntimeError::Protocol)?; + receipt.validate().map_err(RuntimeError::Protocol)?; + receipt_path = self.request_path(&request.unit_id, &request.request_id, true)?; + atomic_write(&receipt_path, &receipt, "request receipt")?; + Ok(reservation(record, receipt)) } fn load_sync(&self, unit_id: &str) -> RuntimeResult { validate_unit_id(unit_id)?; let _lock = self.lock(unit_id)?; - let path = self.record_path(unit_id)?; - if !path.is_file() { - return Err(RuntimeError::NotFound { + let path = self.record_path(unit_id, false)?; + read_required_record(&path, unit_id) + } + + fn load_request_sync( + &self, + unit_id: &str, + request_id: &str, + ) -> RuntimeResult { + validate_unit_id(unit_id)?; + validate_request_id(request_id)?; + let _lock = self.lock(unit_id)?; + let path = self.request_path(unit_id, request_id, false)?; + let receipt = + read_optional_receipt(&path)?.ok_or_else(|| RuntimeError::RequestNotFound { unit_id: unit_id.into(), - }); - } - let record = read_record(&path)?; - if record.spec.unit_id != unit_id { - return Err(RuntimeError::Protocol("Runtime state key mismatch".into())); + request_id: request_id.into(), + })?; + if receipt.unit_id != unit_id || receipt.request_id != request_id { + return Err(RuntimeError::Protocol( + "Runtime request receipt storage key mismatch".into(), + )); } - Ok(record) + Ok(receipt) } fn update_observation_sync( @@ -193,63 +296,120 @@ impl FileRuntimeStateStore { ) -> RuntimeResult { observation.validate().map_err(RuntimeError::Protocol)?; let _lock = self.lock(&observation.unit_id)?; - let path = self.record_path(&observation.unit_id)?; - if !path.is_file() { - return Err(RuntimeError::NotFound { - unit_id: observation.unit_id, - }); - } - let mut record = read_record(&path)?; + let record_path = self.record_path(&observation.unit_id, false)?; + let mut record = read_required_record(&record_path, &observation.unit_id)?; if record.removed_at_ms.is_some() { return Err(RuntimeError::Protocol( "cannot update an explicitly removed Runtime unit".into(), )); } validate_transition(&record.observation, &observation, &record.spec)?; - record.observation = observation.clone(); + if let Some(request_id) = request_id { - let receipt = record - .receipt_mut(&request_id) - .map_err(RuntimeError::Protocol)?; - if receipt.kind == RuntimeRequestKind::Remove { + let receipt_path = self.request_path(&observation.unit_id, &request_id, false)?; + let mut receipt = + read_required_receipt(&receipt_path, &observation.unit_id, &request_id)?; + if !matches!( + receipt.kind, + RuntimeRequestKind::Apply | RuntimeRequestKind::Stop + ) { return Err(RuntimeError::Protocol( - "remove request cannot complete with an observation".into(), + "request kind cannot complete with an observation".into(), )); } - receipt.complete_with_observation(observation); + ensure_receipt_target(&receipt, &observation.unit_id, observation.generation)?; + if receipt.state == RuntimeRequestState::Completed { + if receipt.observation.as_ref() != Some(&observation) { + return Err(RuntimeError::Protocol( + "completed Runtime request result changed".into(), + )); + } + } else { + receipt.complete_with_observation(observation.clone()); + receipt.validate().map_err(RuntimeError::Protocol)?; + atomic_write(&receipt_path, &receipt, "request receipt")?; + } } + + record.observation = observation; record.validate().map_err(RuntimeError::Protocol)?; - atomic_write(&path, &record)?; + atomic_write(&record_path, &record, "state record")?; Ok(record) } fn complete_removal_sync(&self, removal: RuntimeRemoval) -> RuntimeResult { removal.validate().map_err(RuntimeError::Protocol)?; let _lock = self.lock(&removal.unit_id)?; - let path = self.record_path(&removal.unit_id)?; - if !path.is_file() { - return Err(RuntimeError::NotFound { - unit_id: removal.unit_id, - }); - } - let mut record = read_record(&path)?; + let record_path = self.record_path(&removal.unit_id, false)?; + let mut record = read_required_record(&record_path, &removal.unit_id)?; if removal.generation != record.spec.generation { return Err(RuntimeError::Protocol( "Runtime removal generation does not match stored unit".into(), )); } - let receipt = record - .receipt_mut(&removal.request_id) - .map_err(RuntimeError::Protocol)?; + let receipt_path = self.request_path(&removal.unit_id, &removal.request_id, false)?; + let mut receipt = + read_required_receipt(&receipt_path, &removal.unit_id, &removal.request_id)?; if receipt.kind != RuntimeRequestKind::Remove { return Err(RuntimeError::Protocol( "non-remove request completed with removal result".into(), )); } - receipt.complete_with_removal(removal.clone()); + ensure_receipt_target(&receipt, &removal.unit_id, removal.generation)?; + if receipt.state == RuntimeRequestState::Completed { + if receipt.removal.as_ref() != Some(&removal) { + return Err(RuntimeError::Protocol( + "completed Runtime removal result changed".into(), + )); + } + } else { + receipt.complete_with_removal(removal.clone()); + receipt.validate().map_err(RuntimeError::Protocol)?; + atomic_write(&receipt_path, &receipt, "request receipt")?; + } record.removed_at_ms = Some(removal.removed_at_ms); record.validate().map_err(RuntimeError::Protocol)?; - atomic_write(&path, &record)?; + atomic_write(&record_path, &record, "state record")?; + Ok(record) + } + + fn complete_exec_sync(&self, result: RuntimeExecResult) -> RuntimeResult { + result.validate().map_err(RuntimeError::Protocol)?; + let unit_id = result.observation.unit_id.clone(); + let _lock = self.lock(&unit_id)?; + let record_path = self.record_path(&unit_id, false)?; + let mut record = read_required_record(&record_path, &unit_id)?; + if record.removed_at_ms.is_some() { + return Err(RuntimeError::Protocol( + "cannot complete exec for an explicitly removed Runtime unit".into(), + )); + } + result + .observation + .validate_against(&record.spec) + .map_err(RuntimeError::Protocol)?; + validate_transition(&record.observation, &result.observation, &record.spec)?; + let receipt_path = self.request_path(&unit_id, &result.request_id, false)?; + let mut receipt = read_required_receipt(&receipt_path, &unit_id, &result.request_id)?; + if receipt.kind != RuntimeRequestKind::Exec { + return Err(RuntimeError::Protocol( + "non-exec request completed with exec result".into(), + )); + } + if receipt.state == RuntimeRequestState::Completed { + if receipt.exec_result.as_ref() != Some(&result) { + return Err(RuntimeError::Protocol( + "completed Runtime exec result changed".into(), + )); + } + } else { + receipt.complete_with_exec_result(result.clone()); + receipt.validate().map_err(RuntimeError::Protocol)?; + atomic_write(&receipt_path, &receipt, "request receipt")?; + } + record.observation = result.observation; + record.validate().map_err(RuntimeError::Protocol)?; + atomic_write(&record_path, &record, "state record")?; Ok(record) } @@ -267,11 +427,50 @@ impl FileRuntimeStateStore { Ok(StateLock(file)) } - fn record_path(&self, unit_id: &str) -> RuntimeResult { + fn unit_directory(&self, unit_id: &str, create: bool) -> RuntimeResult { validate_unit_id(unit_id)?; - let records = self.root.join("units"); - ensure_directory(&records)?; - Ok(records.join(format!("{}.json", storage_key(unit_id)))) + let units = self.root.join("units"); + if create { + ensure_directory(&self.root)?; + ensure_directory(&units)?; + } else if path_exists(&units)? { + ensure_directory(&units)?; + } + let key = storage_key(unit_id); + let legacy = units.join(format!("{key}.json")); + if path_exists(&legacy)? { + return Err(RuntimeError::Protocol(format!( + "legacy Runtime unit record {} requires explicit migration", + legacy.display() + ))); + } + let unit = units.join(key); + if create { + ensure_directory(&unit)?; + } else if path_exists(&unit)? { + ensure_directory(&unit)?; + } + Ok(unit) + } + + fn record_path(&self, unit_id: &str, create: bool) -> RuntimeResult { + Ok(self.unit_directory(unit_id, create)?.join("record.json")) + } + + fn request_path( + &self, + unit_id: &str, + request_id: &str, + create: bool, + ) -> RuntimeResult { + validate_request_id(request_id)?; + let requests = self.unit_directory(unit_id, create)?.join("requests"); + if create { + ensure_directory(&requests)?; + } else if path_exists(&requests)? { + ensure_directory(&requests)?; + } + Ok(requests.join(format!("{}.json", storage_key(request_id)))) } } @@ -315,6 +514,18 @@ impl RuntimeStateStore for FileRuntimeStateStore { .map_err(task_error)? } + async fn reserve_exec( + &self, + request: &RuntimeExecRequest, + _now_ms: u64, + ) -> RuntimeResult { + let store = self.clone(); + let request = request.clone(); + tokio::task::spawn_blocking(move || store.reserve_exec_sync(request)) + .await + .map_err(task_error)? + } + async fn load(&self, unit_id: &str) -> RuntimeResult { let store = self.clone(); let unit_id = unit_id.to_owned(); @@ -323,6 +534,19 @@ impl RuntimeStateStore for FileRuntimeStateStore { .map_err(task_error)? } + async fn load_request( + &self, + unit_id: &str, + request_id: &str, + ) -> RuntimeResult { + let store = self.clone(); + let unit_id = unit_id.to_owned(); + let request_id = request_id.to_owned(); + tokio::task::spawn_blocking(move || store.load_request_sync(&unit_id, &request_id)) + .await + .map_err(task_error)? + } + async fn update_observation( &self, request_id: Option<&str>, @@ -343,21 +567,25 @@ impl RuntimeStateStore for FileRuntimeStateStore { .await .map_err(task_error)? } + + async fn complete_exec(&self, result: &RuntimeExecResult) -> RuntimeResult { + let store = self.clone(); + let result = result.clone(); + tokio::task::spawn_blocking(move || store.complete_exec_sync(result)) + .await + .map_err(task_error)? + } } fn reservation( record: RuntimeUnitRecord, - request_id: &str, -) -> RuntimeResult { - let receipt = record - .receipt(request_id) - .cloned() - .ok_or_else(|| RuntimeError::Protocol("reserved request receipt is missing".into()))?; - Ok(RuntimeStateReservation { + receipt: RuntimeRequestReceipt, +) -> RuntimeStateReservation { + RuntimeStateReservation { dispatch: receipt.state == RuntimeRequestState::Pending, record, receipt, - }) + } } fn ensure_same_request( @@ -373,6 +601,104 @@ fn ensure_same_request( Ok(()) } +fn ensure_receipt_target( + receipt: &RuntimeRequestReceipt, + unit_id: &str, + generation: u64, +) -> RuntimeResult<()> { + if receipt.unit_id != unit_id || receipt.generation != generation { + return Err(RuntimeError::RequestConflict { + request_id: receipt.request_id.clone(), + }); + } + Ok(()) +} + +fn ensure_current_generation( + record: &RuntimeUnitRecord, + unit_id: &str, + requested: u64, +) -> RuntimeResult<()> { + if requested < record.spec.generation { + return Err(RuntimeError::StaleGeneration { + unit_id: unit_id.into(), + requested, + current: record.spec.generation, + }); + } + if requested != record.spec.generation { + return Err(RuntimeError::GenerationConflict { + unit_id: unit_id.into(), + generation: requested, + }); + } + Ok(()) +} + +fn reconcile_completed_observation( + record: &mut RuntimeUnitRecord, + receipt: &RuntimeRequestReceipt, +) -> RuntimeResult { + let Some(observation) = &receipt.observation else { + return Ok(false); + }; + if record.removed_at_ms.is_some() + || record.spec.generation != receipt.generation + || record.spec.unit_id != receipt.unit_id + || record.observation == *observation + || record.observation.observed_at_ms > observation.observed_at_ms + { + return Ok(false); + } + if validate_transition(&record.observation, observation, &record.spec).is_ok() { + record.observation = observation.clone(); + record.validate().map_err(RuntimeError::Protocol)?; + return Ok(true); + } + Ok(false) +} + +fn reconcile_completed_removal( + record: &mut RuntimeUnitRecord, + receipt: &RuntimeRequestReceipt, +) -> RuntimeResult { + let Some(removal) = &receipt.removal else { + return Ok(false); + }; + if record.spec.unit_id == receipt.unit_id + && record.spec.generation == receipt.generation + && record.removed_at_ms.is_none() + { + record.removed_at_ms = Some(removal.removed_at_ms); + record.validate().map_err(RuntimeError::Protocol)?; + return Ok(true); + } + Ok(false) +} + +fn reconcile_completed_exec( + record: &mut RuntimeUnitRecord, + receipt: &RuntimeRequestReceipt, +) -> RuntimeResult { + let Some(result) = &receipt.exec_result else { + return Ok(false); + }; + if record.removed_at_ms.is_some() + || record.spec.generation != receipt.generation + || record.spec.unit_id != receipt.unit_id + || record.observation == result.observation + || record.observation.observed_at_ms > result.observation.observed_at_ms + { + return Ok(false); + } + if validate_transition(&record.observation, &result.observation, &record.spec).is_ok() { + record.observation = result.observation.clone(); + record.validate().map_err(RuntimeError::Protocol)?; + return Ok(true); + } + Ok(false) +} + pub(crate) fn validate_transition( current: &RuntimeObservation, next: &RuntimeObservation, @@ -464,16 +790,21 @@ impl Drop for FileOperationLease { } } -fn storage_key(unit_id: &str) -> String { - format!("{:x}", Sha256::digest(unit_id.as_bytes())) +fn storage_key(value: &str) -> String { + format!("{:x}", Sha256::digest(value.as_bytes())) } fn validate_unit_id(unit_id: &str) -> RuntimeResult<()> { crate::contract::validate_id("unit_id", unit_id, 512).map_err(RuntimeError::InvalidRequest) } +fn validate_request_id(request_id: &str) -> RuntimeResult<()> { + crate::contract::validate_id("request_id", request_id, 512) + .map_err(RuntimeError::InvalidRequest) +} + fn ensure_directory(path: &Path) -> RuntimeResult<()> { - if path.exists() { + if path_exists(path)? { let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state path"))?; if metadata.file_type().is_symlink() || !metadata.is_dir() { return Err(RuntimeError::Protocol(format!( @@ -481,6 +812,7 @@ fn ensure_directory(path: &Path) -> RuntimeResult<()> { path.display() ))); } + verify_owner(&metadata, path, "state directory")?; } else { std::fs::create_dir_all(path).map_err(io_error("create state directory"))?; } @@ -512,14 +844,65 @@ fn owner_only_open(path: &Path, label: &str) -> RuntimeResult { Ok(file) } -fn read_record(path: &Path) -> RuntimeResult { - let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state record"))?; - if metadata.file_type().is_symlink() || !metadata.is_file() { +fn read_required_record(path: &Path, unit_id: &str) -> RuntimeResult { + let record = read_optional_record(path)?.ok_or_else(|| RuntimeError::NotFound { + unit_id: unit_id.into(), + })?; + if record.spec.unit_id != unit_id { + return Err(RuntimeError::Protocol("Runtime state key mismatch".into())); + } + Ok(record) +} + +fn read_optional_record(path: &Path) -> RuntimeResult> { + read_optional_json(path, MAX_RECORD_BYTES, "state record") +} + +fn read_required_receipt( + path: &Path, + unit_id: &str, + request_id: &str, +) -> RuntimeResult { + let receipt = read_optional_receipt(path)?.ok_or_else(|| RuntimeError::RequestNotFound { + unit_id: unit_id.into(), + request_id: request_id.into(), + })?; + if receipt.unit_id != unit_id || receipt.request_id != request_id { return Err(RuntimeError::Protocol( - "Runtime state record is not a regular file".into(), + "Runtime request receipt storage key mismatch".into(), )); } - let mut bytes = Vec::new(); + Ok(receipt) +} + +fn read_optional_receipt(path: &Path) -> RuntimeResult> { + let receipt: Option = + read_optional_json(path, MAX_RECEIPT_BYTES, "request receipt")?; + if let Some(receipt) = &receipt { + receipt.validate().map_err(RuntimeError::Protocol)?; + } + Ok(receipt) +} + +fn read_optional_json( + path: &Path, + max_bytes: u64, + label: &str, +) -> RuntimeResult> { + if !regular_file_exists(path, label)? { + return Ok(None); + } + let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state file"))?; + verify_owner_only_file(&metadata, path, label)?; + if metadata.len() > max_bytes { + return Err(RuntimeError::Protocol(format!( + "Runtime {label} exceeds its size limit" + ))); + } + let capacity = usize::try_from(metadata.len()).map_err(|_| { + RuntimeError::Protocol(format!("Runtime {label} size cannot be represented")) + })?; + let mut bytes = Vec::with_capacity(capacity); let mut options = OpenOptions::new(); options.read(true); #[cfg(unix)] @@ -530,11 +913,32 @@ fn read_record(path: &Path) -> RuntimeResult { options .open(path) .and_then(|mut file| file.read_to_end(&mut bytes)) - .map_err(io_error("read state record"))?; - let record: RuntimeUnitRecord = serde_json::from_slice(&bytes) - .map_err(|error| RuntimeError::Protocol(format!("invalid state record: {error}")))?; - record.validate().map_err(RuntimeError::Protocol)?; - Ok(record) + .map_err(io_error("read state file"))?; + let value = serde_json::from_slice(&bytes) + .map_err(|error| RuntimeError::Protocol(format!("invalid {label}: {error}")))?; + Ok(Some(value)) +} + +fn regular_file_exists(path: &Path, label: &str) -> RuntimeResult { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + Err(RuntimeError::Protocol(format!( + "Runtime {label} {} is not a regular file", + path.display() + ))) + } + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(io_error("inspect state file")(error)), + } +} + +fn path_exists(path: &Path) -> RuntimeResult { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(io_error("inspect state path")(error)), + } } fn reject_symlink(path: &Path, label: &str) -> RuntimeResult<()> { @@ -549,29 +953,65 @@ fn reject_symlink(path: &Path, label: &str) -> RuntimeResult<()> { } } -fn atomic_write(path: &Path, record: &RuntimeUnitRecord) -> RuntimeResult<()> { +#[cfg(unix)] +fn verify_owner(metadata: &std::fs::Metadata, path: &Path, label: &str) -> RuntimeResult<()> { + use std::os::unix::fs::MetadataExt; + if metadata.uid() != unsafe { libc::geteuid() } { + return Err(RuntimeError::Protocol(format!( + "Runtime {label} {} is owned by another user", + path.display() + ))); + } + Ok(()) +} + +#[cfg(not(unix))] +fn verify_owner(_metadata: &std::fs::Metadata, _path: &Path, _label: &str) -> RuntimeResult<()> { + Ok(()) +} + +fn verify_owner_only_file( + metadata: &std::fs::Metadata, + path: &Path, + label: &str, +) -> RuntimeResult<()> { + verify_owner(metadata, path, label)?; + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + if metadata.permissions().mode() & 0o077 != 0 || metadata.nlink() != 1 { + return Err(RuntimeError::Protocol(format!( + "Runtime {label} {} is not an owner-only unlinked file", + path.display() + ))); + } + } + Ok(()) +} + +fn atomic_write(path: &Path, value: &T, label: &str) -> RuntimeResult<()> { let parent = path .parent() - .ok_or_else(|| RuntimeError::Protocol("state record has no parent".into()))?; - let bytes = serde_json::to_vec(record) - .map_err(|error| RuntimeError::Protocol(format!("encode state record: {error}")))?; + .ok_or_else(|| RuntimeError::Protocol(format!("{label} has no parent")))?; + let bytes = serde_json::to_vec(value) + .map_err(|error| RuntimeError::Protocol(format!("encode {label}: {error}")))?; let mut temporary = tempfile::NamedTempFile::new_in(parent).map_err(io_error("create state staging file"))?; temporary .write_all(&bytes) .and_then(|()| temporary.as_file().sync_all()) - .map_err(io_error("write state record"))?; + .map_err(io_error("write state file"))?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; temporary .as_file() .set_permissions(std::fs::Permissions::from_mode(0o600)) - .map_err(io_error("secure state record"))?; + .map_err(io_error("secure state file"))?; } temporary .persist(path) - .map_err(|error| io_error("publish state record")(error.error))?; + .map_err(|error| io_error("publish state file")(error.error))?; #[cfg(unix)] File::open(parent) .and_then(|directory| directory.sync_all()) diff --git a/src/state/mod.rs b/src/state/mod.rs index f4bd29d..e8df649 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -2,7 +2,8 @@ mod file; mod record; use crate::contract::{ - RuntimeActionRequest, RuntimeApplyRequest, RuntimeObservation, RuntimeRemoval, + RuntimeActionRequest, RuntimeApplyRequest, RuntimeExecRequest, RuntimeExecResult, + RuntimeObservation, RuntimeRemoval, }; use crate::RuntimeResult; use async_trait::async_trait; @@ -37,8 +38,20 @@ pub trait RuntimeStateStore: Send + Sync { now_ms: u64, ) -> RuntimeResult; + async fn reserve_exec( + &self, + request: &RuntimeExecRequest, + now_ms: u64, + ) -> RuntimeResult; + async fn load(&self, unit_id: &str) -> RuntimeResult; + async fn load_request( + &self, + unit_id: &str, + request_id: &str, + ) -> RuntimeResult; + async fn update_observation( &self, request_id: Option<&str>, @@ -46,4 +59,6 @@ pub trait RuntimeStateStore: Send + Sync { ) -> RuntimeResult; async fn complete_removal(&self, removal: &RuntimeRemoval) -> RuntimeResult; + + async fn complete_exec(&self, result: &RuntimeExecResult) -> RuntimeResult; } diff --git a/src/state/record.rs b/src/state/record.rs index 18adc67..e18f15c 100644 --- a/src/state/record.rs +++ b/src/state/record.rs @@ -1,8 +1,8 @@ use crate::contract::{ - RuntimeActionRequest, RuntimeApplyRequest, RuntimeObservation, RuntimeRemoval, RuntimeUnitSpec, + RuntimeActionRequest, RuntimeApplyRequest, RuntimeExecRequest, RuntimeExecResult, + RuntimeObservation, RuntimeRemoval, RuntimeUnitSpec, }; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -17,6 +17,7 @@ pub enum RuntimeRequestKind { Apply, Stop, Remove, + Exec, } impl From for RuntimeRequestKind { @@ -38,75 +39,166 @@ pub enum RuntimeRequestState { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeRequestReceipt { + pub schema: String, pub request_id: String, + pub unit_id: String, + pub generation: u64, pub kind: RuntimeRequestKind, pub request_digest: String, pub state: RuntimeRequestState, pub observation: Option, pub removal: Option, + pub exec_result: Option, } impl RuntimeRequestReceipt { + pub const SCHEMA: &'static str = "a3s.runtime.request-receipt.v1"; + pub(crate) fn pending_apply(request: &RuntimeApplyRequest) -> Result { - Ok(Self { - request_id: request.request_id.clone(), - kind: RuntimeRequestKind::Apply, - request_digest: request.digest()?, - state: RuntimeRequestState::Pending, - observation: None, - removal: None, - }) + Ok(Self::pending( + request.request_id.clone(), + request.spec.unit_id.clone(), + request.spec.generation, + RuntimeRequestKind::Apply, + request.digest()?, + )) } pub(crate) fn pending_action( kind: RuntimeActionKind, request: &RuntimeActionRequest, ) -> Result { - Ok(Self { - request_id: request.request_id.clone(), - kind: kind.into(), - request_digest: request.digest()?, + Ok(Self::pending( + request.request_id.clone(), + request.unit_id.clone(), + request.generation, + kind.into(), + request.digest()?, + )) + } + + pub(crate) fn pending_exec(request: &RuntimeExecRequest) -> Result { + Ok(Self::pending( + request.request_id.clone(), + request.unit_id.clone(), + request.generation, + RuntimeRequestKind::Exec, + request.digest()?, + )) + } + + fn pending( + request_id: String, + unit_id: String, + generation: u64, + kind: RuntimeRequestKind, + request_digest: String, + ) -> Self { + Self { + schema: Self::SCHEMA.into(), + request_id, + unit_id, + generation, + kind, + request_digest, state: RuntimeRequestState::Pending, observation: None, removal: None, - }) + exec_result: None, + } } pub(crate) fn complete_with_observation(&mut self, observation: RuntimeObservation) { self.state = RuntimeRequestState::Completed; self.observation = Some(observation); self.removal = None; + self.exec_result = None; } pub(crate) fn complete_with_removal(&mut self, removal: RuntimeRemoval) { self.state = RuntimeRequestState::Completed; self.observation = None; self.removal = Some(removal); + self.exec_result = None; + } + + pub(crate) fn complete_with_exec_result(&mut self, result: RuntimeExecResult) { + self.state = RuntimeRequestState::Completed; + self.observation = None; + self.removal = None; + self.exec_result = Some(result); } - fn validate(&self) -> Result<(), String> { + pub fn validate(&self) -> Result<(), String> { + if self.schema != Self::SCHEMA { + return Err(format!( + "unsupported Runtime request receipt schema {:?}", + self.schema + )); + } crate::contract::validate_id("request_id", &self.request_id, 512)?; + crate::contract::validate_id("unit_id", &self.unit_id, 512)?; + if self.generation == 0 { + return Err("Runtime request receipt generation must be positive".into()); + } crate::contract::validate_digest(&self.request_digest)?; - match (self.kind, self.state) { - (_, RuntimeRequestState::Pending) - if self.observation.is_none() && self.removal.is_none() => - { - Ok(()) - } + match ( + self.kind, + self.state, + &self.observation, + &self.removal, + &self.exec_result, + ) { + (_, RuntimeRequestState::Pending, None, None, None) => Ok(()), ( RuntimeRequestKind::Apply | RuntimeRequestKind::Stop, RuntimeRequestState::Completed, - ) if self.observation.is_some() && self.removal.is_none() => { - self.observation.as_ref().unwrap().validate() + Some(observation), + None, + None, + ) => { + observation.validate()?; + self.validate_unit_result(&observation.unit_id, observation.generation) } - (RuntimeRequestKind::Remove, RuntimeRequestState::Completed) - if self.observation.is_none() && self.removal.is_some() => - { - self.removal.as_ref().unwrap().validate() + ( + RuntimeRequestKind::Remove, + RuntimeRequestState::Completed, + None, + Some(removal), + None, + ) => { + removal.validate()?; + if removal.request_id != self.request_id { + return Err("Runtime removal receipt request identity mismatch".into()); + } + self.validate_unit_result(&removal.unit_id, removal.generation) + } + ( + RuntimeRequestKind::Exec, + RuntimeRequestState::Completed, + None, + None, + Some(result), + ) => { + result.validate()?; + if result.request_id != self.request_id { + return Err("Runtime exec receipt request identity mismatch".into()); + } + self.validate_unit_result( + &result.observation.unit_id, + result.observation.generation, + ) } _ => Err("Runtime request receipt result does not match its kind and state".into()), } } + + fn validate_unit_result(&self, unit_id: &str, generation: u64) -> Result<(), String> { + if unit_id != self.unit_id || generation != self.generation { + return Err("Runtime request receipt result identity mismatch".into()); + } + Ok(()) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -116,22 +208,17 @@ pub struct RuntimeUnitRecord { pub spec: RuntimeUnitSpec, pub observation: RuntimeObservation, pub removed_at_ms: Option, - pub requests: BTreeMap, } impl RuntimeUnitRecord { - pub const SCHEMA: &'static str = "a3s.runtime.unit-record.v1"; + pub const SCHEMA: &'static str = "a3s.runtime.unit-record.v2"; pub(crate) fn new(request: &RuntimeApplyRequest, now_ms: u64) -> Result { - let receipt = RuntimeRequestReceipt::pending_apply(request)?; - let mut requests = BTreeMap::new(); - requests.insert(request.request_id.clone(), receipt); let record = Self { schema: Self::SCHEMA.into(), spec: request.spec.clone(), observation: RuntimeObservation::accepted(&request.spec, now_ms)?, removed_at_ms: None, - requests, }; record.validate()?; Ok(record) @@ -145,40 +232,7 @@ impl RuntimeUnitRecord { )); } self.spec.validate()?; - self.observation.validate_against(&self.spec)?; - if self.requests.len() > 10_000 { - return Err("Runtime unit record exceeds request receipt limit".into()); - } - for (request_id, receipt) in &self.requests { - receipt.validate()?; - if request_id != &receipt.request_id { - return Err("Runtime request receipt key mismatch".into()); - } - if let Some(observation) = &receipt.observation { - if observation.unit_id != self.spec.unit_id { - return Err("Runtime request receipt belongs to another unit".into()); - } - } - if let Some(removal) = &receipt.removal { - if removal.unit_id != self.spec.unit_id { - return Err("Runtime removal receipt belongs to another unit".into()); - } - } - } - Ok(()) - } - - pub fn receipt(&self, request_id: &str) -> Option<&RuntimeRequestReceipt> { - self.requests.get(request_id) - } - - pub(crate) fn receipt_mut( - &mut self, - request_id: &str, - ) -> Result<&mut RuntimeRequestReceipt, String> { - self.requests - .get_mut(request_id) - .ok_or_else(|| format!("Runtime request receipt {request_id:?} was not reserved")) + self.observation.validate_against(&self.spec) } } diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 9df782c..ff53089 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -139,10 +139,26 @@ impl RuntimeStateStore for SignalingStateStore { self.inner.reserve_action(kind, request, now_ms).await } + async fn reserve_exec( + &self, + request: &RuntimeExecRequest, + now_ms: u64, + ) -> RuntimeResult { + self.inner.reserve_exec(request, now_ms).await + } + async fn load(&self, unit_id: &str) -> RuntimeResult { self.inner.load(unit_id).await } + async fn load_request( + &self, + unit_id: &str, + request_id: &str, + ) -> RuntimeResult { + self.inner.load_request(unit_id, request_id).await + } + async fn update_observation( &self, request_id: Option<&str>, @@ -154,6 +170,10 @@ impl RuntimeStateStore for SignalingStateStore { async fn complete_removal(&self, removal: &RuntimeRemoval) -> RuntimeResult { self.inner.complete_removal(removal).await } + + async fn complete_exec(&self, result: &RuntimeExecResult) -> RuntimeResult { + self.inner.complete_exec(result).await + } } #[async_trait] @@ -519,7 +539,11 @@ async fn ambiguous_apply_is_reentered_with_the_same_identity() { let pending = store.load("service-ambiguous").await.unwrap(); assert_eq!(pending.observation.state, RuntimeUnitState::Accepted); assert_eq!( - pending.receipt("apply-ambiguous").unwrap().state, + store + .load_request("service-ambiguous", "apply-ambiguous") + .await + .unwrap() + .state, a3s_runtime::RuntimeRequestState::Pending ); @@ -855,6 +879,47 @@ async fn exec_result_must_bind_the_complete_unit_spec() { )); } +#[tokio::test] +async fn completed_exec_replays_durably_after_client_restart_and_rejects_conflicts() { + let driver = Arc::new(TestDriver::new()); + let (client, store) = driver.client(); + client + .apply(&apply("exec-restart-apply", service("exec-restart", 1))) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-restart-request".into(), + unit_id: "exec-restart".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }; + let expected = client.exec(&request).await.unwrap(); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + assert_eq!( + store + .load_request("exec-restart", "exec-restart-request") + .await + .unwrap() + .exec_result, + Some(expected.clone()) + ); + + let restarted = ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + assert_eq!(restarted.exec(&request).await.unwrap(), expected); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + + let mut conflicting = request; + conflicting.command = vec!["/bin/false".into()]; + assert!(matches!( + restarted.exec(&conflicting).await, + Err(RuntimeError::RequestConflict { .. }) + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); +} + #[test] fn top_level_log_exec_and_inspection_records_require_current_schemas() { let mut query = RuntimeLogQuery { @@ -980,6 +1045,22 @@ async fn log_and_exec_capabilities_use_the_active_generation() { .await .unwrap(); assert_eq!(result.exit_code, 0); + assert_eq!( + client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-tools".into(), + unit_id: "service-tools".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }) + .await + .unwrap(), + result + ); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); assert!(matches!( client @@ -1176,11 +1257,11 @@ async fn deadlines_bound_capability_queries_and_provider_dispatch() { client.apply(&request).await, Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider apply") )); - let pending = store.load("provider-timeout").await.unwrap(); - assert_eq!( - pending.receipt("provider-timeout").unwrap().state, - a3s_runtime::RuntimeRequestState::Pending - ); + let pending = store + .load_request("provider-timeout", "provider-timeout") + .await + .unwrap(); + assert_eq!(pending.state, a3s_runtime::RuntimeRequestState::Pending); assert_eq!(provider_driver.apply_calls.load(Ordering::SeqCst), 1); provider_driver.hang_apply.store(false, Ordering::SeqCst); @@ -1334,11 +1415,58 @@ async fn concurrent_file_reservations_preserve_every_request() { let record = store.load("concurrent-service").await.unwrap(); assert_eq!(record.spec, expected); - assert_eq!(record.requests.len(), 32); - assert!(record - .requests - .values() - .all(|receipt| receipt.state == a3s_runtime::RuntimeRequestState::Pending)); + for index in 0..32 { + assert_eq!( + store + .load_request("concurrent-service", &format!("apply-{index}")) + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); + } +} + +#[tokio::test] +async fn request_journal_preserves_more_than_ten_thousand_exact_replays() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let client = + ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), Arc::new(FixedClock)); + let spec = service("journal-capacity", 1); + client + .apply(&apply("journal-request-0", spec.clone())) + .await + .unwrap(); + for index in 1..=10_000 { + client + .apply(&apply(&format!("journal-request-{index}"), spec.clone())) + .await + .unwrap(); + } + + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); + assert_eq!( + store + .load_request("journal-capacity", "journal-request-10000") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Completed + ); + let unit_directory = std::fs::read_dir(directory.path().join("units")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + assert_eq!( + std::fs::read_dir(unit_directory.join("requests")) + .unwrap() + .count(), + 10_001 + ); } struct CountingFactory { @@ -1446,6 +1574,91 @@ async fn file_state_store_rejects_symbolic_link_boundaries() { )); } +#[cfg(unix)] +#[tokio::test] +async fn request_journal_layout_is_versioned_owner_only_and_fail_closed() { + use sha2::{Digest, Sha256}; + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let client = ManagedRuntimeClient::with_clock(store.clone(), driver, Arc::new(FixedClock)); + client + .apply(&apply("layout-request", service("layout-unit", 1))) + .await + .unwrap(); + + let unit_directory = std::fs::read_dir(directory.path().join("units")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let record_path = unit_directory.join("record.json"); + let receipt_path = std::fs::read_dir(unit_directory.join("requests")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let record: serde_json::Value = + serde_json::from_slice(&std::fs::read(&record_path).unwrap()).unwrap(); + let receipt: serde_json::Value = + serde_json::from_slice(&std::fs::read(&receipt_path).unwrap()).unwrap(); + assert_eq!(record["schema"], a3s_runtime::RuntimeUnitRecord::SCHEMA); + assert!(record.get("requests").is_none()); + assert_eq!( + receipt["schema"], + a3s_runtime::RuntimeRequestReceipt::SCHEMA + ); + assert_eq!( + std::fs::metadata(&unit_directory) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); + assert_eq!( + std::fs::metadata(&record_path) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + assert_eq!( + std::fs::metadata(&receipt_path) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + + let mut unknown_field = receipt; + unknown_field["unexpected"] = serde_json::json!(true); + std::fs::write(&receipt_path, serde_json::to_vec(&unknown_field).unwrap()).unwrap(); + assert!(matches!( + store.load_request("layout-unit", "layout-request").await, + Err(RuntimeError::Protocol(_)) + )); + + let legacy = tempfile::tempdir().unwrap(); + let units = legacy.path().join("units"); + std::fs::create_dir(&units).unwrap(); + let key = format!("{:x}", Sha256::digest(b"legacy-unit")); + let legacy_record = units.join(format!("{key}.json")); + std::fs::write(&legacy_record, b"{}").unwrap(); + std::fs::set_permissions(&legacy_record, std::fs::Permissions::from_mode(0o600)).unwrap(); + let legacy_store = FileRuntimeStateStore::new(legacy.path()); + assert!(matches!( + legacy_store.load("legacy-unit").await, + Err(RuntimeError::Protocol(message)) if message.contains("explicit migration") + )); +} + #[tokio::test] async fn exported_conformance_suite_exercises_task_and_service_lifecycles() { let driver = Arc::new(TestDriver::new()); From 1a26e65300fa5ad02299799af10eacd6075ee093 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 12:31:33 +0800 Subject: [PATCH 05/32] refactor: simplify journal path validation --- src/state/file.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/state/file.rs b/src/state/file.rs index fb79ff6..c535a2a 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -445,9 +445,7 @@ impl FileRuntimeStateStore { ))); } let unit = units.join(key); - if create { - ensure_directory(&unit)?; - } else if path_exists(&unit)? { + if create || path_exists(&unit)? { ensure_directory(&unit)?; } Ok(unit) @@ -465,9 +463,7 @@ impl FileRuntimeStateStore { ) -> RuntimeResult { validate_request_id(request_id)?; let requests = self.unit_directory(unit_id, create)?.join("requests"); - if create { - ensure_directory(&requests)?; - } else if path_exists(&requests)? { + if create || path_exists(&requests)? { ensure_directory(&requests)?; } Ok(requests.join(format!("{}.json", storage_key(request_id)))) From d74d0ac75c8a3b14deaa16951bf869f0b61a910b Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 12:43:28 +0800 Subject: [PATCH 06/32] test: freeze runtime wire protocol fixtures --- README.md | 10 +- ...mplete-protocol-and-operation-semantics.md | 5 +- src/contract/unit.rs | 2 +- tests/golden/action-request-v1.json | 7 + tests/golden/apply-request-v1.json | 42 ++++ tests/golden/capabilities-v3.json | 13 ++ tests/golden/exec-request-v1.json | 9 + tests/golden/exec-result-v1.json | 31 +++ tests/golden/inspection-found-v1.json | 27 +++ tests/golden/inspection-not-found-v1.json | 6 + tests/golden/log-chunk-v1.json | 8 + tests/golden/log-query-v1.json | 8 + tests/golden/observation-v2.json | 49 +++++ tests/golden/removal-v1.json | 8 + tests/golden/request-receipt-v1.json | 42 ++++ tests/golden/unit-record-v2.json | 60 ++++++ tests/golden/unit-spec-v2.json | 97 ++++++++++ tests/protocol_golden.rs | 180 ++++++++++++++++++ 18 files changed, 600 insertions(+), 4 deletions(-) create mode 100644 tests/golden/action-request-v1.json create mode 100644 tests/golden/apply-request-v1.json create mode 100644 tests/golden/capabilities-v3.json create mode 100644 tests/golden/exec-request-v1.json create mode 100644 tests/golden/exec-result-v1.json create mode 100644 tests/golden/inspection-found-v1.json create mode 100644 tests/golden/inspection-not-found-v1.json create mode 100644 tests/golden/log-chunk-v1.json create mode 100644 tests/golden/log-query-v1.json create mode 100644 tests/golden/observation-v2.json create mode 100644 tests/golden/removal-v1.json create mode 100644 tests/golden/request-receipt-v1.json create mode 100644 tests/golden/unit-record-v2.json create mode 100644 tests/golden/unit-spec-v2.json create mode 100644 tests/protocol_golden.rs diff --git a/README.md b/README.md index 6840d38..956a9d0 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,8 @@ A unit specification includes: - an optional digest binding caller-owned execution semantics. All wire records use explicit schema identifiers and reject unknown fields. +`RuntimeUnitSpec` v2 makes the ephemeral-storage quota optional; a provider +needs that resource control only when a specification requests the quota. Protocol validation occurs before state reservation or provider work. ### Observations @@ -132,8 +134,12 @@ provider integration: ```text state root/ -├── locks/ # per-unit cross-process locks -└── units/ # atomic JSON records and request receipts +├── locks/ # short per-unit record locks +├── operations/ # full-operation cross-process leases +└── units/ + └── / + ├── record.json # active unit record + └── requests/ # one durable receipt per request ID ``` The store uses a SHA-256 storage key derived from the validated unit ID. Records diff --git a/docs/adr/0002-complete-protocol-and-operation-semantics.md b/docs/adr/0002-complete-protocol-and-operation-semantics.md index 52b7f0d..f2ecace 100644 --- a/docs/adr/0002-complete-protocol-and-operation-semantics.md +++ b/docs/adr/0002-complete-protocol-and-operation-semantics.md @@ -44,7 +44,10 @@ fields: - durable unit records and request receipts. Nested value objects remain versioned by their enclosing top-level schema. -Adding `size_bytes` to output artifacts changes observations to +Making `ephemeral_storage_bytes` optional changes unit specifications to +`a3s.runtime.unit-spec.v2`; v1 required a numeric quota. An omitted quota is +encoded explicitly as `null` in v2 and requires no ephemeral-storage +capability. Adding `size_bytes` to output artifacts changes observations to `a3s.runtime.observation.v2`. Adding typed provider identity and output capability changes capabilities to `a3s.runtime.capabilities.v3`. The durable request-journal layout uses `a3s.runtime.unit-record.v2`. diff --git a/src/contract/unit.rs b/src/contract/unit.rs index e335f49..b39bbb0 100644 --- a/src/contract/unit.rs +++ b/src/contract/unit.rs @@ -226,7 +226,7 @@ pub struct RuntimeUnitSpec { } impl RuntimeUnitSpec { - pub const SCHEMA: &'static str = "a3s.runtime.unit-spec.v1"; + pub const SCHEMA: &'static str = "a3s.runtime.unit-spec.v2"; pub fn validate(&self) -> Result<(), String> { if self.schema != Self::SCHEMA { diff --git a/tests/golden/action-request-v1.json b/tests/golden/action-request-v1.json new file mode 100644 index 0000000..a12568d --- /dev/null +++ b/tests/golden/action-request-v1.json @@ -0,0 +1,7 @@ +{ + "schema": "a3s.runtime.action-request.v1", + "request_id": "golden-stop", + "unit_id": "golden-service", + "generation": 3, + "deadline_at_ms": 200000 +} diff --git a/tests/golden/apply-request-v1.json b/tests/golden/apply-request-v1.json new file mode 100644 index 0000000..8b2fc78 --- /dev/null +++ b/tests/golden/apply-request-v1.json @@ -0,0 +1,42 @@ +{ + "schema": "a3s.runtime.apply-request.v1", + "request_id": "golden-apply", + "deadline_at_ms": 200000, + "spec": { + "schema": "a3s.runtime.unit-spec.v2", + "unit_id": "golden-task", + "generation": 7, + "class": "task", + "artifact": { + "uri": "oci://registry.example/a3s/golden@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "media_type": "application/vnd.oci.image.manifest.v1+json" + }, + "process": { + "command": [], + "args": [], + "working_directory": null, + "environment": {} + }, + "mounts": [], + "secrets": [], + "network": { + "mode": "outbound", + "ports": [] + }, + "resources": { + "cpu_millis": 500, + "memory_bytes": 134217728, + "pids": 128, + "ephemeral_storage_bytes": null, + "execution_timeout_ms": 60000 + }, + "isolation": "container", + "health": null, + "restart": { + "kind": "never" + }, + "outputs": [], + "semantics_profile_digest": null + } +} diff --git a/tests/golden/capabilities-v3.json b/tests/golden/capabilities-v3.json new file mode 100644 index 0000000..c6cb8ae --- /dev/null +++ b/tests/golden/capabilities-v3.json @@ -0,0 +1,13 @@ +{ + "schema": "a3s.runtime.capabilities.v3", + "provider_id": "golden-runtime", + "provider_build": "golden-runtime/1.2.3", + "unit_classes": ["task", "service"], + "artifact_media_types": ["application/vnd.oci.image.manifest.v1+json"], + "isolation_levels": ["process", "container", "sandbox", "confidential"], + "network_modes": ["none", "outbound", "service"], + "mount_kinds": ["artifact", "volume", "tmpfs"], + "health_check_kinds": ["http", "tcp", "command"], + "resource_controls": ["cpu", "memory", "pids", "ephemeral_storage", "execution_timeout"], + "features": ["durable_identity", "stop", "remove", "logs", "exec", "usage", "attestation", "secret_references", "output_artifacts"] +} diff --git a/tests/golden/exec-request-v1.json b/tests/golden/exec-request-v1.json new file mode 100644 index 0000000..dab9aef --- /dev/null +++ b/tests/golden/exec-request-v1.json @@ -0,0 +1,9 @@ +{ + "schema": "a3s.runtime.exec-request.v1", + "request_id": "golden-exec", + "unit_id": "golden-service", + "generation": 3, + "command": ["/bin/sh", "-c", "printf ready"], + "timeout_ms": 5000, + "deadline_at_ms": 200000 +} diff --git a/tests/golden/exec-result-v1.json b/tests/golden/exec-result-v1.json new file mode 100644 index 0000000..f5e4002 --- /dev/null +++ b/tests/golden/exec-result-v1.json @@ -0,0 +1,31 @@ +{ + "schema": "a3s.runtime.exec-result.v1", + "request_id": "golden-exec", + "observation": { + "schema": "a3s.runtime.observation.v2", + "unit_id": "golden-service", + "generation": 3, + "spec_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "class": "service", + "state": "running", + "provider_resource_id": "golden/service/3", + "provider_build": "golden-runtime/1.2.3", + "observed_at_ms": 120000, + "started_at_ms": 100000, + "finished_at_ms": null, + "health": { + "state": "healthy", + "checked_at_ms": 119000, + "message": "ready" + }, + "outputs": [], + "usage": null, + "evidence": null, + "provider_attestation": null, + "failure": null + }, + "exit_code": 0, + "stdout": "ready", + "stderr": "", + "truncated": false +} diff --git a/tests/golden/inspection-found-v1.json b/tests/golden/inspection-found-v1.json new file mode 100644 index 0000000..c63bba2 --- /dev/null +++ b/tests/golden/inspection-found-v1.json @@ -0,0 +1,27 @@ +{ + "status": "found", + "schema": "a3s.runtime.inspection.v1", + "observation": { + "schema": "a3s.runtime.observation.v2", + "unit_id": "golden-service", + "generation": 3, + "spec_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "class": "service", + "state": "running", + "provider_resource_id": "golden/service/3", + "provider_build": "golden-runtime/1.2.3", + "observed_at_ms": 120000, + "started_at_ms": 100000, + "finished_at_ms": null, + "health": { + "state": "healthy", + "checked_at_ms": 119000, + "message": "ready" + }, + "outputs": [], + "usage": null, + "evidence": null, + "provider_attestation": null, + "failure": null + } +} diff --git a/tests/golden/inspection-not-found-v1.json b/tests/golden/inspection-not-found-v1.json new file mode 100644 index 0000000..b4f12b5 --- /dev/null +++ b/tests/golden/inspection-not-found-v1.json @@ -0,0 +1,6 @@ +{ + "status": "not_found", + "schema": "a3s.runtime.inspection.v1", + "unit_id": "golden-service", + "last_generation": 3 +} diff --git a/tests/golden/log-chunk-v1.json b/tests/golden/log-chunk-v1.json new file mode 100644 index 0000000..2a97847 --- /dev/null +++ b/tests/golden/log-chunk-v1.json @@ -0,0 +1,8 @@ +{ + "schema": "a3s.runtime.log-chunk.v1", + "cursor": "cursor-42", + "sequence": 42, + "observed_at_ms": 120000, + "stream": "stderr", + "data": "warning\n" +} diff --git a/tests/golden/log-query-v1.json b/tests/golden/log-query-v1.json new file mode 100644 index 0000000..5b5ad95 --- /dev/null +++ b/tests/golden/log-query-v1.json @@ -0,0 +1,8 @@ +{ + "schema": "a3s.runtime.log-query.v1", + "unit_id": "golden-service", + "generation": 3, + "cursor": "cursor-41", + "limit": 100, + "stream": "stderr" +} diff --git a/tests/golden/observation-v2.json b/tests/golden/observation-v2.json new file mode 100644 index 0000000..fadf034 --- /dev/null +++ b/tests/golden/observation-v2.json @@ -0,0 +1,49 @@ +{ + "schema": "a3s.runtime.observation.v2", + "unit_id": "golden-task", + "generation": 7, + "spec_digest": "sha256:3309c80ee072c47311be878a7fcde9818e484e9d186fa1c6eb78033fe5ef4d11", + "class": "task", + "state": "succeeded", + "provider_resource_id": "golden/task/7", + "provider_build": "golden-runtime/1.2.3", + "observed_at_ms": 120000, + "started_at_ms": 100000, + "finished_at_ms": 120000, + "health": null, + "outputs": [ + { + "name": "result", + "artifact": { + "uri": "artifact://outputs/result@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "media_type": "application/json" + }, + "size_bytes": 1024 + } + ], + "usage": { + "wall_time_ms": 20000, + "cpu_time_ms": 10000, + "peak_memory_bytes": 67108864, + "network_rx_bytes": 1024, + "network_tx_bytes": 2048, + "storage_read_bytes": 4096, + "storage_write_bytes": 8192 + }, + "evidence": { + "provider_build": "golden-runtime/1.2.3", + "spec_digest": "sha256:3309c80ee072c47311be878a7fcde9818e484e9d186fa1c6eb78033fe5ef4d11", + "semantics_profile_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "claims": { + "isolation": "sandbox", + "runtime": "golden-runtime" + } + }, + "provider_attestation": { + "uri": "attestation://golden/task/7@sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "media_type": "application/vnd.a3s.attestation.v1+json" + }, + "failure": null +} diff --git a/tests/golden/removal-v1.json b/tests/golden/removal-v1.json new file mode 100644 index 0000000..029ee77 --- /dev/null +++ b/tests/golden/removal-v1.json @@ -0,0 +1,8 @@ +{ + "schema": "a3s.runtime.removal.v1", + "request_id": "golden-remove", + "unit_id": "golden-service", + "generation": 3, + "removed_at_ms": 150000, + "already_absent": false +} diff --git a/tests/golden/request-receipt-v1.json b/tests/golden/request-receipt-v1.json new file mode 100644 index 0000000..7cde408 --- /dev/null +++ b/tests/golden/request-receipt-v1.json @@ -0,0 +1,42 @@ +{ + "schema": "a3s.runtime.request-receipt.v1", + "request_id": "golden-exec", + "unit_id": "golden-service", + "generation": 3, + "kind": "exec", + "request_digest": "sha256:861f6cad44261c49a89b5cd12920ca3977d833b5c9143d748db945a08939bad0", + "state": "completed", + "observation": null, + "removal": null, + "exec_result": { + "schema": "a3s.runtime.exec-result.v1", + "request_id": "golden-exec", + "observation": { + "schema": "a3s.runtime.observation.v2", + "unit_id": "golden-service", + "generation": 3, + "spec_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "class": "service", + "state": "running", + "provider_resource_id": "golden/service/3", + "provider_build": "golden-runtime/1.2.3", + "observed_at_ms": 120000, + "started_at_ms": 100000, + "finished_at_ms": null, + "health": { + "state": "healthy", + "checked_at_ms": 119000, + "message": "ready" + }, + "outputs": [], + "usage": null, + "evidence": null, + "provider_attestation": null, + "failure": null + }, + "exit_code": 0, + "stdout": "ready", + "stderr": "", + "truncated": false + } +} diff --git a/tests/golden/unit-record-v2.json b/tests/golden/unit-record-v2.json new file mode 100644 index 0000000..c6fbb02 --- /dev/null +++ b/tests/golden/unit-record-v2.json @@ -0,0 +1,60 @@ +{ + "schema": "a3s.runtime.unit-record.v2", + "spec": { + "schema": "a3s.runtime.unit-spec.v2", + "unit_id": "golden-task", + "generation": 7, + "class": "task", + "artifact": { + "uri": "oci://registry.example/a3s/golden@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "media_type": "application/vnd.oci.image.manifest.v1+json" + }, + "process": { + "command": [], + "args": [], + "working_directory": null, + "environment": {} + }, + "mounts": [], + "secrets": [], + "network": { + "mode": "outbound", + "ports": [] + }, + "resources": { + "cpu_millis": 500, + "memory_bytes": 134217728, + "pids": 128, + "ephemeral_storage_bytes": null, + "execution_timeout_ms": 60000 + }, + "isolation": "container", + "health": null, + "restart": { + "kind": "never" + }, + "outputs": [], + "semantics_profile_digest": null + }, + "observation": { + "schema": "a3s.runtime.observation.v2", + "unit_id": "golden-task", + "generation": 7, + "spec_digest": "sha256:3309c80ee072c47311be878a7fcde9818e484e9d186fa1c6eb78033fe5ef4d11", + "class": "task", + "state": "accepted", + "provider_resource_id": null, + "provider_build": null, + "observed_at_ms": 100000, + "started_at_ms": null, + "finished_at_ms": null, + "health": null, + "outputs": [], + "usage": null, + "evidence": null, + "provider_attestation": null, + "failure": null + }, + "removed_at_ms": null +} diff --git a/tests/golden/unit-spec-v2.json b/tests/golden/unit-spec-v2.json new file mode 100644 index 0000000..eeb1513 --- /dev/null +++ b/tests/golden/unit-spec-v2.json @@ -0,0 +1,97 @@ +{ + "schema": "a3s.runtime.unit-spec.v2", + "unit_id": "golden-task", + "generation": 7, + "class": "task", + "artifact": { + "uri": "oci://registry.example/a3s/golden@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "media_type": "application/vnd.oci.image.manifest.v1+json" + }, + "process": { + "command": ["/bin/sh"], + "args": ["-c", "echo ready"], + "working_directory": "/workspace", + "environment": { + "LANG": "C.UTF-8" + } + }, + "mounts": [ + { + "name": "source", + "source": { + "kind": "artifact", + "artifact": { + "uri": "artifact://source@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "media_type": "application/vnd.a3s.source.v1+tar" + } + }, + "target": "/inputs/source", + "read_only": true + }, + { + "name": "cache", + "source": { + "kind": "volume", + "volume_id": "cache-volume" + }, + "target": "/cache", + "read_only": false + }, + { + "name": "scratch", + "source": { + "kind": "tmpfs", + "size_bytes": 1048576 + }, + "target": "/tmp", + "read_only": false + } + ], + "secrets": [ + { + "name": "api token", + "reference": "secret://runtime/api-token", + "target": { + "kind": "environment", + "variable": "API_TOKEN" + } + }, + { + "name": "certificate", + "reference": "secret://runtime/certificate", + "target": { + "kind": "file", + "path": "/run/secrets/cert.pem", + "mode": 384 + } + } + ], + "network": { + "mode": "outbound", + "ports": [] + }, + "resources": { + "cpu_millis": 500, + "memory_bytes": 134217728, + "pids": 128, + "ephemeral_storage_bytes": null, + "execution_timeout_ms": 60000 + }, + "isolation": "sandbox", + "health": null, + "restart": { + "kind": "on_failure", + "max_retries": 2 + }, + "outputs": [ + { + "name": "result", + "path": "/outputs/result.json", + "media_type": "application/json", + "max_bytes": 4096 + } + ], + "semantics_profile_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" +} diff --git a/tests/protocol_golden.rs b/tests/protocol_golden.rs new file mode 100644 index 0000000..cb6595a --- /dev/null +++ b/tests/protocol_golden.rs @@ -0,0 +1,180 @@ +use a3s_runtime::contract::{ + RuntimeActionRequest, RuntimeApplyRequest, RuntimeCapabilities, RuntimeExecRequest, + RuntimeExecResult, RuntimeInspection, RuntimeLogChunk, RuntimeLogQuery, RuntimeObservation, + RuntimeRemoval, RuntimeUnitSpec, +}; +use a3s_runtime::{RuntimeRequestReceipt, RuntimeUnitRecord}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use serde_json::{Map, Value}; + +fn adjacent_schema(schema: &str, offset: i64) -> String { + let (prefix, version) = schema + .rsplit_once(".v") + .expect("versioned schema must end in .v"); + let version = version.parse::().expect("schema version must be numeric"); + format!("{prefix}.v{}", version + offset) +} + +fn object(value: &mut Value) -> &mut Map { + value + .as_object_mut() + .expect("every top-level Runtime wire fixture must be an object") +} + +fn assert_top_level_fixture( + raw: &str, + expected_schema: &str, + validate: impl Fn(&T) -> Result<(), String>, +) where + T: DeserializeOwned + Serialize, +{ + let value: Value = serde_json::from_str(raw).expect("golden fixture must be valid JSON"); + assert_eq!(value.get("schema"), Some(&Value::String(expected_schema.into()))); + + let decoded: T = + serde_json::from_value(value.clone()).expect("current golden fixture must decode"); + validate(&decoded).expect("current golden fixture must validate"); + assert_eq!( + serde_json::to_value(&decoded).expect("golden value must encode"), + value, + "golden encode must preserve the complete public wire shape" + ); + + let mut missing = value.clone(); + object(&mut missing).remove("schema"); + assert!( + serde_json::from_value::(missing).is_err(), + "a missing schema must fail closed" + ); + + for incompatible_schema in [ + adjacent_schema(expected_schema, -1), + adjacent_schema(expected_schema, 1), + ] { + let mut incompatible = value.clone(); + object(&mut incompatible).insert("schema".into(), incompatible_schema.into()); + let decoded: T = serde_json::from_value(incompatible) + .expect("a syntactically valid schema string must decode before validation"); + assert!( + validate(&decoded).is_err(), + "old and future schemas must fail validation" + ); + } + + let mut malformed = value.clone(); + object(&mut malformed).insert("schema".into(), Value::Bool(true)); + assert!( + serde_json::from_value::(malformed).is_err(), + "a non-string schema must fail decoding" + ); + + let mut unknown = value; + object(&mut unknown).insert("unexpected".into(), Value::Bool(true)); + assert!( + serde_json::from_value::(unknown).is_err(), + "unknown top-level fields must fail closed" + ); +} + +#[test] +fn ct_schema_001_every_top_level_wire_record_has_a_versioned_golden_fixture() { + assert_top_level_fixture::( + include_str!("golden/capabilities-v3.json"), + RuntimeCapabilities::SCHEMA, + RuntimeCapabilities::validate, + ); + assert_top_level_fixture::( + include_str!("golden/unit-spec-v2.json"), + RuntimeUnitSpec::SCHEMA, + RuntimeUnitSpec::validate, + ); + assert_top_level_fixture::( + include_str!("golden/apply-request-v1.json"), + RuntimeApplyRequest::SCHEMA, + RuntimeApplyRequest::validate, + ); + assert_top_level_fixture::( + include_str!("golden/action-request-v1.json"), + RuntimeActionRequest::SCHEMA, + RuntimeActionRequest::validate, + ); + assert_top_level_fixture::( + include_str!("golden/observation-v2.json"), + RuntimeObservation::SCHEMA, + RuntimeObservation::validate, + ); + for inspection in [ + include_str!("golden/inspection-found-v1.json"), + include_str!("golden/inspection-not-found-v1.json"), + ] { + assert_top_level_fixture::( + inspection, + RuntimeInspection::SCHEMA, + RuntimeInspection::validate, + ); + } + assert_top_level_fixture::( + include_str!("golden/removal-v1.json"), + RuntimeRemoval::SCHEMA, + RuntimeRemoval::validate, + ); + assert_top_level_fixture::( + include_str!("golden/log-query-v1.json"), + RuntimeLogQuery::SCHEMA, + RuntimeLogQuery::validate, + ); + assert_top_level_fixture::( + include_str!("golden/log-chunk-v1.json"), + RuntimeLogChunk::SCHEMA, + RuntimeLogChunk::validate, + ); + assert_top_level_fixture::( + include_str!("golden/exec-request-v1.json"), + RuntimeExecRequest::SCHEMA, + RuntimeExecRequest::validate, + ); + assert_top_level_fixture::( + include_str!("golden/exec-result-v1.json"), + RuntimeExecResult::SCHEMA, + RuntimeExecResult::validate, + ); + assert_top_level_fixture::( + include_str!("golden/unit-record-v2.json"), + RuntimeUnitRecord::SCHEMA, + RuntimeUnitRecord::validate, + ); + assert_top_level_fixture::( + include_str!("golden/request-receipt-v1.json"), + RuntimeRequestReceipt::SCHEMA, + RuntimeRequestReceipt::validate, + ); +} + +#[test] +fn ct_digest_001_golden_request_and_spec_digests_are_stable() { + let unit: RuntimeUnitSpec = + serde_json::from_str(include_str!("golden/unit-spec-v2.json")).unwrap(); + assert_eq!( + unit.digest().unwrap(), + "sha256:56d815af40b2bcb083a0fd8b42bcf4501330362d81b3850a45d901725d828610" + ); + + let apply: RuntimeApplyRequest = + serde_json::from_str(include_str!("golden/apply-request-v1.json")).unwrap(); + assert_eq!( + apply.spec.digest().unwrap(), + "sha256:3309c80ee072c47311be878a7fcde9818e484e9d186fa1c6eb78033fe5ef4d11" + ); + assert_eq!( + apply.digest().unwrap(), + "sha256:e178a719503c68db140756ae8568c3e51da35843de1133c3a6585520f0f4ea91" + ); + + let exec: RuntimeExecRequest = + serde_json::from_str(include_str!("golden/exec-request-v1.json")).unwrap(); + assert_eq!( + exec.digest().unwrap(), + "sha256:861f6cad44261c49a89b5cd12920ca3977d833b5c9143d748db945a08939bad0" + ); +} From b258d8d21596d6e563fb013acdfcf2a6071396b3 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 12:45:08 +0800 Subject: [PATCH 07/32] style: format protocol golden tests --- tests/protocol_golden.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/protocol_golden.rs b/tests/protocol_golden.rs index cb6595a..2c92fc6 100644 --- a/tests/protocol_golden.rs +++ b/tests/protocol_golden.rs @@ -12,7 +12,9 @@ fn adjacent_schema(schema: &str, offset: i64) -> String { let (prefix, version) = schema .rsplit_once(".v") .expect("versioned schema must end in .v"); - let version = version.parse::().expect("schema version must be numeric"); + let version = version + .parse::() + .expect("schema version must be numeric"); format!("{prefix}.v{}", version + offset) } @@ -30,7 +32,10 @@ fn assert_top_level_fixture( T: DeserializeOwned + Serialize, { let value: Value = serde_json::from_str(raw).expect("golden fixture must be valid JSON"); - assert_eq!(value.get("schema"), Some(&Value::String(expected_schema.into()))); + assert_eq!( + value.get("schema"), + Some(&Value::String(expected_schema.into())) + ); let decoded: T = serde_json::from_value(value.clone()).expect("current golden fixture must decode"); From 420ae4c48e9dde412dfe2ad319fb5c90436dc79f Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 12:49:26 +0800 Subject: [PATCH 08/32] fix: replay completed requests beyond deadlines --- README.md | 6 +- ...mplete-protocol-and-operation-semantics.md | 8 + src/managed.rs | 140 +++++++++++++++++- tests/general_runtime.rs | 97 ++++++++++++ 4 files changed, 248 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 956a9d0..da1f504 100644 --- a/README.md +++ b/README.md @@ -112,8 +112,10 @@ The `RuntimeClient` contract exposes: Each mutating request carries its own request ID and optional absolute deadline. An exact retry returns or reconstructs the same logical result. Reusing a -request ID with different content fails with `RequestConflict`. A deadline is -checked independently before provider dispatch. +request ID with different content fails with `RequestConflict`. A completed +receipt remains replayable after its original deadline and after later +lifecycle operations; an expired pending request is not redispatched. A +deadline is checked independently before new provider work. ## Capabilities diff --git a/docs/adr/0002-complete-protocol-and-operation-semantics.md b/docs/adr/0002-complete-protocol-and-operation-semantics.md index f2ecace..32a15fa 100644 --- a/docs/adr/0002-complete-protocol-and-operation-semantics.md +++ b/docs/adr/0002-complete-protocol-and-operation-semantics.md @@ -198,6 +198,14 @@ acquiring the operation lease, and use the remaining duration to bound the provider future. A timeout returns `DeadlineExceeded` and leaves a dispatched mutating request pending because provider acknowledgement is ambiguous. +An exact replay whose receipt is already `completed` returns that durable +result before capability or deadline checks, including after the original +absolute deadline and after later lifecycle operations. The core reacquires +the unit lease and lets the state store reconcile a receipt-first crash before +returning. Deadlines constrain unfinished work; they do not invalidate an +already committed response. A pending replay remains subject to its original +deadline and is never redispatched after that deadline expires. + Drivers may enforce a shorter provider-specific timeout. They must never extend the caller deadline. Exec uses the smaller of its relative `timeout_ms` and an optional absolute request deadline. diff --git a/src/managed.rs b/src/managed.rs index 2e948c3..5779656 100644 --- a/src/managed.rs +++ b/src/managed.rs @@ -5,7 +5,8 @@ use crate::contract::{ }; use crate::{ RuntimeActionKind, RuntimeClient, RuntimeClock, RuntimeDriver, RuntimeError, RuntimeResult, - RuntimeStateStore, SystemRuntimeClock, + RuntimeRequestKind, RuntimeRequestReceipt, RuntimeRequestState, RuntimeStateStore, + SystemRuntimeClock, }; use async_trait::async_trait; use std::future::Future; @@ -95,6 +96,36 @@ impl ManagedRuntimeClient { .deadline_at_ms .map_or(relative, |absolute| absolute.min(relative)) } + + async fn completed_replay( + &self, + unit_id: &str, + generation: u64, + request_id: &str, + kind: RuntimeRequestKind, + request_digest: &str, + ) -> RuntimeResult> { + let receipt = match self.state.load_request(unit_id, request_id).await { + Ok(receipt) => receipt, + Err(RuntimeError::RequestNotFound { .. }) => return Ok(None), + Err(error) => return Err(error), + }; + receipt.validate().map_err(RuntimeError::Protocol)?; + if receipt.unit_id != unit_id || receipt.request_id != request_id { + return Err(RuntimeError::Protocol( + "Runtime request receipt storage key mismatch".into(), + )); + } + if receipt.generation != generation + || receipt.kind != kind + || receipt.request_digest != request_digest + { + return Err(RuntimeError::RequestConflict { + request_id: request_id.into(), + }); + } + Ok((receipt.state == RuntimeRequestState::Completed).then_some(receipt)) + } } #[async_trait] @@ -105,6 +136,35 @@ impl RuntimeClient for ManagedRuntimeClient { async fn apply(&self, request: &RuntimeApplyRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; + let request_digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + if self + .completed_replay( + &request.spec.unit_id, + request.spec.generation, + &request.request_id, + RuntimeRequestKind::Apply, + &request_digest, + ) + .await? + .is_some() + { + let _lease = self + .state + .acquire_operation_lease(&request.spec.unit_id) + .await?; + let reservation = self + .state + .reserve_apply(request, self.clock.now_ms()) + .await?; + if reservation.dispatch { + return Err(RuntimeError::Protocol( + "completed apply receipt regressed to pending".into(), + )); + } + return reservation.receipt.observation.ok_or_else(|| { + RuntimeError::Protocol("completed apply receipt has no observation".into()) + }); + } self.check_deadline(request.deadline_at_ms)?; let capabilities = self .bounded( @@ -220,6 +280,35 @@ impl RuntimeClient for ManagedRuntimeClient { async fn stop(&self, request: &RuntimeActionRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; + let request_digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + if self + .completed_replay( + &request.unit_id, + request.generation, + &request.request_id, + RuntimeRequestKind::Stop, + &request_digest, + ) + .await? + .is_some() + { + let _lease = self.state.acquire_operation_lease(&request.unit_id).await?; + let reservation = self + .state + .reserve_action(RuntimeActionKind::Stop, request, self.clock.now_ms()) + .await?; + if reservation.dispatch { + return Err(RuntimeError::Protocol( + "completed stop receipt regressed to pending".into(), + )); + } + return Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), + observation: Box::new(reservation.receipt.observation.ok_or_else(|| { + RuntimeError::Protocol("completed stop receipt has no observation".into()) + })?), + }); + } self.check_deadline(request.deadline_at_ms)?; let capabilities = self .bounded( @@ -276,6 +365,32 @@ impl RuntimeClient for ManagedRuntimeClient { async fn remove(&self, request: &RuntimeActionRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; + let request_digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + if self + .completed_replay( + &request.unit_id, + request.generation, + &request.request_id, + RuntimeRequestKind::Remove, + &request_digest, + ) + .await? + .is_some() + { + let _lease = self.state.acquire_operation_lease(&request.unit_id).await?; + let reservation = self + .state + .reserve_action(RuntimeActionKind::Remove, request, self.clock.now_ms()) + .await?; + if reservation.dispatch { + return Err(RuntimeError::Protocol( + "completed remove receipt regressed to pending".into(), + )); + } + return reservation.receipt.removal.ok_or_else(|| { + RuntimeError::Protocol("completed remove receipt has no removal".into()) + }); + } self.check_deadline(request.deadline_at_ms)?; let capabilities = self .bounded( @@ -354,6 +469,29 @@ impl RuntimeClient for ManagedRuntimeClient { async fn exec(&self, request: &RuntimeExecRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; + let request_digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + if self + .completed_replay( + &request.unit_id, + request.generation, + &request.request_id, + RuntimeRequestKind::Exec, + &request_digest, + ) + .await? + .is_some() + { + let _lease = self.state.acquire_operation_lease(&request.unit_id).await?; + let reservation = self.state.reserve_exec(request, self.clock.now_ms()).await?; + if reservation.dispatch { + return Err(RuntimeError::Protocol( + "completed exec receipt regressed to pending".into(), + )); + } + return reservation.receipt.exec_result.ok_or_else(|| { + RuntimeError::Protocol("completed exec receipt has no result".into()) + }); + } let deadline_at_ms = Some(self.exec_deadline(request)); self.check_deadline(deadline_at_ms)?; let capabilities = self diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index ff53089..2d219bb 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -920,6 +920,103 @@ async fn completed_exec_replays_durably_after_client_restart_and_rejects_conflic assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); } +#[tokio::test] +async fn completed_mutations_replay_after_deadline_provider_loss_and_removal() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let clock = Arc::new(ManualClock::new(NOW)); + let client = + ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); + let deadline = NOW + 1_000; + + let mut apply_request = apply("durable-replay-apply", service("durable-replay", 1)); + apply_request.deadline_at_ms = Some(deadline); + let applied = client.apply(&apply_request).await.unwrap(); + + let exec_request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "durable-replay-exec".into(), + unit_id: "durable-replay".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 500, + deadline_at_ms: Some(deadline), + }; + let executed = client.exec(&exec_request).await.unwrap(); + + let mut stop_request = action("durable-replay-stop", "durable-replay", 1); + stop_request.deadline_at_ms = Some(deadline); + let stopped = client.stop(&stop_request).await.unwrap(); + + let mut remove_request = action("durable-replay-remove", "durable-replay", 1); + remove_request.deadline_at_ms = Some(deadline); + let removed = client.remove(&remove_request).await.unwrap(); + + clock.set(deadline); + driver.hang_capabilities.store(true, Ordering::SeqCst); + let restarted = ManagedRuntimeClient::with_clock(store, driver.clone(), clock); + + assert_eq!(restarted.apply(&apply_request).await.unwrap(), applied); + assert_eq!(restarted.exec(&exec_request).await.unwrap(), executed); + assert_eq!(restarted.stop(&stop_request).await.unwrap(), stopped); + assert_eq!(restarted.remove(&remove_request).await.unwrap(), removed); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + assert_eq!(driver.stop_calls.load(Ordering::SeqCst), 1); + assert_eq!(driver.remove_calls.load(Ordering::SeqCst), 1); + + let mut conflicting = apply_request; + conflicting.spec.process.command = vec!["/bin/false".into()]; + assert!(matches!( + restarted.apply(&conflicting).await, + Err(RuntimeError::RequestConflict { .. }) + )); +} + +#[tokio::test] +async fn expired_pending_request_remains_pending_and_is_not_redispatched() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let clock = Arc::new(ManualClock::new(NOW)); + let client = + ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); + let mut request = apply("pending-expiry", service("pending-expiry", 1)); + request.deadline_at_ms = Some(NOW + 10); + driver.hang_apply.store(true, Ordering::SeqCst); + + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider apply") + )); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); + assert_eq!( + store + .load_request("pending-expiry", "pending-expiry") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); + + clock.set(NOW + 10); + driver.hang_apply.store(false, Ordering::SeqCst); + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(_)) + )); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); + assert_eq!( + store + .load_request("pending-expiry", "pending-expiry") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); +} + #[test] fn top_level_log_exec_and_inspection_records_require_current_schemas() { let mut query = RuntimeLogQuery { From a7b8d019c6a8834ab617fe592d78ac631af288e4 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 12:50:22 +0800 Subject: [PATCH 09/32] style: format completed replay paths --- src/managed.rs | 11 +++++++---- tests/general_runtime.rs | 6 ++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/managed.rs b/src/managed.rs index 5779656..9e0ac30 100644 --- a/src/managed.rs +++ b/src/managed.rs @@ -4,9 +4,9 @@ use crate::contract::{ RuntimeRemoval, RuntimeUnitState, }; use crate::{ - RuntimeActionKind, RuntimeClient, RuntimeClock, RuntimeDriver, RuntimeError, RuntimeResult, - RuntimeRequestKind, RuntimeRequestReceipt, RuntimeRequestState, RuntimeStateStore, - SystemRuntimeClock, + RuntimeActionKind, RuntimeClient, RuntimeClock, RuntimeDriver, RuntimeError, + RuntimeRequestKind, RuntimeRequestReceipt, RuntimeRequestState, RuntimeResult, + RuntimeStateStore, SystemRuntimeClock, }; use async_trait::async_trait; use std::future::Future; @@ -482,7 +482,10 @@ impl RuntimeClient for ManagedRuntimeClient { .is_some() { let _lease = self.state.acquire_operation_lease(&request.unit_id).await?; - let reservation = self.state.reserve_exec(request, self.clock.now_ms()).await?; + let reservation = self + .state + .reserve_exec(request, self.clock.now_ms()) + .await?; if reservation.dispatch { return Err(RuntimeError::Protocol( "completed exec receipt regressed to pending".into(), diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 2d219bb..7253642 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -926,8 +926,7 @@ async fn completed_mutations_replay_after_deadline_provider_loss_and_removal() { let store = Arc::new(FileRuntimeStateStore::new(directory.path())); let driver = Arc::new(TestDriver::new()); let clock = Arc::new(ManualClock::new(NOW)); - let client = - ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); + let client = ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); let deadline = NOW + 1_000; let mut apply_request = apply("durable-replay-apply", service("durable-replay", 1)); @@ -980,8 +979,7 @@ async fn expired_pending_request_remains_pending_and_is_not_redispatched() { let store = Arc::new(FileRuntimeStateStore::new(directory.path())); let driver = Arc::new(TestDriver::new()); let clock = Arc::new(ManualClock::new(NOW)); - let client = - ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); + let client = ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); let mut request = apply("pending-expiry", service("pending-expiry", 1)); request.deadline_at_ms = Some(NOW + 10); driver.hang_apply.store(true, Ordering::SeqCst); From 0608ab4d4716c464a4b341010a1f1abbc8a3ade9 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 12:53:12 +0800 Subject: [PATCH 10/32] test: scope runtime state fixture directories --- tests/general_runtime.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 7253642..4feda47 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -16,7 +16,7 @@ use a3s_runtime::{ use async_trait::async_trait; use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::Duration; use tokio::sync::Notify; @@ -54,6 +54,7 @@ impl RuntimeClock for ManualClock { struct TestDriver { provider: ProviderId, capabilities: RuntimeCapabilities, + state_directories: Mutex>, apply_calls: AtomicUsize, inspect_calls: AtomicUsize, stop_calls: AtomicUsize, @@ -78,6 +79,7 @@ impl TestDriver { Self { capabilities: capabilities(provider.clone()), provider, + state_directories: Mutex::new(Vec::new()), apply_calls: AtomicUsize::new(0), inspect_calls: AtomicUsize::new(0), stop_calls: AtomicUsize::new(0), @@ -99,7 +101,8 @@ impl TestDriver { fn client(self: &Arc) -> (ManagedRuntimeClient, Arc) { let directory = tempfile::tempdir().unwrap(); - let path = directory.keep(); + let path = directory.path().to_path_buf(); + self.state_directories.lock().unwrap().push(directory); let store = Arc::new(FileRuntimeStateStore::new(path)); let client = ManagedRuntimeClient::with_clock(store.clone(), self.clone(), Arc::new(FixedClock)); From 33d8b35c54f34d113a64b6fc101e0e09eaf1a2ee Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 13:18:33 +0800 Subject: [PATCH 11/32] test: prove generation reconciliation recovery --- README.md | 7 +- src/driver.rs | 4 + tests/general_runtime.rs | 211 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index da1f504..f497ac6 100644 --- a/README.md +++ b/README.md @@ -174,8 +174,11 @@ ManagedRuntimeClient Provider `apply` must be idempotent for the supplied unit ID and generation. After an ambiguous acknowledgement, a repeated call must discover or converge -the same resource rather than create another one. Provider-specific labels, -SDK handles, container fields, and transport details stay behind the driver. +the same resource rather than create another one. A successful generation +handoff must retire all older provider generations and verify that exactly the +current resource remains; interrupted handoffs finish on exact retry. +Provider-specific labels, SDK handles, container fields, and transport details +stay behind the driver. ## Conformance diff --git a/src/driver.rs b/src/driver.rs index 284fb26..b66a94e 100644 --- a/src/driver.rs +++ b/src/driver.rs @@ -11,6 +11,10 @@ use async_trait::async_trait; /// Drivers do not own request idempotency or durable shared state. `apply`, /// `stop`, and `remove` must nevertheless be safe to reattach after an /// ambiguous transport failure using the stable unit identity and generation. +/// A successful `apply` must also retire every older provider generation for +/// the unit before returning, leaving exactly one managed provider resource. +/// If that handoff is interrupted, an exact retry must discover the partially +/// created current generation and finish the same reconciliation. #[async_trait] pub trait RuntimeDriver: Send + Sync { fn provider_id(&self) -> &ProviderId; diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 4feda47..44cc4f2 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -110,6 +110,139 @@ impl TestDriver { } } +struct GenerationDriver { + provider: ProviderId, + resources: Mutex>>, + fail_after_create: AtomicBool, + apply_calls: AtomicUsize, +} + +impl GenerationDriver { + fn new() -> Self { + Self { + provider: ProviderId::parse("generation-runtime").unwrap(), + resources: Mutex::new(BTreeMap::new()), + fail_after_create: AtomicBool::new(false), + apply_calls: AtomicUsize::new(0), + } + } + + fn generations(&self, unit_id: &str) -> Vec { + self.resources + .lock() + .unwrap() + .get(unit_id) + .map(|resources| resources.keys().copied().collect()) + .unwrap_or_default() + } +} + +#[async_trait] +impl RuntimeDriver for GenerationDriver { + fn provider_id(&self) -> &ProviderId { + &self.provider + } + + async fn capabilities(&self) -> RuntimeResult { + Ok(capabilities(self.provider.clone())) + } + + async fn apply( + &self, + spec: &RuntimeUnitSpec, + current: &RuntimeObservation, + ) -> RuntimeResult { + self.apply_calls.fetch_add(1, Ordering::SeqCst); + let resource_id = { + let mut inventory = self.resources.lock().unwrap(); + inventory + .entry(spec.unit_id.clone()) + .or_default() + .entry(spec.generation) + .or_insert_with(|| format!("provider/{}/g{}", spec.unit_id, spec.generation)) + .clone() + }; + if self.fail_after_create.swap(false, Ordering::SeqCst) { + return Err(RuntimeError::Transport( + "ambiguous generation handoff".into(), + )); + } + self.resources + .lock() + .unwrap() + .entry(spec.unit_id.clone()) + .or_default() + .retain(|generation, _| *generation == spec.generation); + + let mut observation = current.clone(); + observation.state = match spec.class { + RuntimeUnitClass::Task => RuntimeUnitState::Succeeded, + RuntimeUnitClass::Service => RuntimeUnitState::Running, + }; + observation.provider_resource_id = Some(resource_id); + observation.provider_build = Some("generation-driver/1".into()); + observation.observed_at_ms = NOW + spec.generation; + observation.started_at_ms = Some(NOW); + observation.finished_at_ms = + (spec.class == RuntimeUnitClass::Task).then_some(observation.observed_at_ms); + observation.health = spec.health.as_ref().map(|_| RuntimeHealthObservation { + state: RuntimeHealthState::Healthy, + checked_at_ms: observation.observed_at_ms, + message: None, + }); + observation + .validate_against(spec) + .map_err(RuntimeError::Protocol)?; + Ok(observation) + } + + async fn inspect(&self, _unit: &RuntimeUnitRecord) -> RuntimeResult { + Err(RuntimeError::Protocol( + "generation test did not expect inspect".into(), + )) + } + + async fn stop( + &self, + _unit: &RuntimeUnitRecord, + _request: &RuntimeActionRequest, + ) -> RuntimeResult { + Err(RuntimeError::Protocol( + "generation test did not expect stop".into(), + )) + } + + async fn remove( + &self, + _unit: &RuntimeUnitRecord, + _request: &RuntimeActionRequest, + ) -> RuntimeResult { + Err(RuntimeError::Protocol( + "generation test did not expect remove".into(), + )) + } + + async fn logs( + &self, + _unit: &RuntimeUnitRecord, + _query: &RuntimeLogQuery, + ) -> RuntimeResult> { + Err(RuntimeError::Protocol( + "generation test did not expect logs".into(), + )) + } + + async fn exec( + &self, + _unit: &RuntimeUnitRecord, + _request: &RuntimeExecRequest, + ) -> RuntimeResult { + Err(RuntimeError::Protocol( + "generation test did not expect exec".into(), + )) + } +} + struct SignalingStateStore { inner: Arc, lease_started: Arc, @@ -592,6 +725,84 @@ async fn request_and_generation_conflicts_fail_closed() { )); } +#[tokio::test] +async fn generation_reconciliation_retires_the_previous_resource_on_success() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(GenerationDriver::new()); + let client = + ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + + client + .apply(&apply( + "generation-success-one", + service("generation-success", 1), + )) + .await + .unwrap(); + assert_eq!(driver.generations("generation-success"), vec![1]); + + let observation = client + .apply(&apply( + "generation-success-two", + service("generation-success", 2), + )) + .await + .unwrap(); + assert_eq!(observation.generation, 2); + assert_eq!(driver.generations("generation-success"), vec![2]); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn generation_reconciliation_recovers_create_before_error_after_restart() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(GenerationDriver::new()); + let client = ManagedRuntimeClient::with_clock( + store.clone(), + driver.clone(), + Arc::new(FixedClock), + ); + client + .apply(&apply( + "generation-recovery-one", + service("generation-recovery", 1), + )) + .await + .unwrap(); + + let second = apply( + "generation-recovery-two", + service("generation-recovery", 2), + ); + driver.fail_after_create.store(true, Ordering::SeqCst); + assert!(matches!( + client.apply(&second).await, + Err(RuntimeError::Transport(message)) if message.contains("ambiguous generation handoff") + )); + assert_eq!(driver.generations("generation-recovery"), vec![1, 2]); + assert_eq!( + store + .load_request("generation-recovery", "generation-recovery-two") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); + + drop(client); + let restarted = + ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + let recovered = restarted.apply(&second).await.unwrap(); + assert_eq!(recovered.generation, 2); + assert_eq!(driver.generations("generation-recovery"), vec![2]); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 3); + + assert_eq!(restarted.apply(&second).await.unwrap(), recovered); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 3); +} + #[tokio::test] async fn stop_remove_and_absence_are_durable_and_idempotent() { let driver = Arc::new(TestDriver::new()); From 45bc439aec51565565976097ffd9884fc1440577 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 13:19:15 +0800 Subject: [PATCH 12/32] style: format generation recovery tests --- tests/general_runtime.rs | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 44cc4f2..642e348 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -730,8 +730,7 @@ async fn generation_reconciliation_retires_the_previous_resource_on_success() { let directory = tempfile::tempdir().unwrap(); let store = Arc::new(FileRuntimeStateStore::new(directory.path())); let driver = Arc::new(GenerationDriver::new()); - let client = - ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + let client = ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); client .apply(&apply( @@ -759,11 +758,8 @@ async fn generation_reconciliation_recovers_create_before_error_after_restart() let directory = tempfile::tempdir().unwrap(); let store = Arc::new(FileRuntimeStateStore::new(directory.path())); let driver = Arc::new(GenerationDriver::new()); - let client = ManagedRuntimeClient::with_clock( - store.clone(), - driver.clone(), - Arc::new(FixedClock), - ); + let client = + ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), Arc::new(FixedClock)); client .apply(&apply( "generation-recovery-one", @@ -772,10 +768,7 @@ async fn generation_reconciliation_recovers_create_before_error_after_restart() .await .unwrap(); - let second = apply( - "generation-recovery-two", - service("generation-recovery", 2), - ); + let second = apply("generation-recovery-two", service("generation-recovery", 2)); driver.fail_after_create.store(true, Ordering::SeqCst); assert!(matches!( client.apply(&second).await, @@ -792,8 +785,7 @@ async fn generation_reconciliation_recovers_create_before_error_after_restart() ); drop(client); - let restarted = - ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + let restarted = ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); let recovered = restarted.apply(&second).await.unwrap(); assert_eq!(recovered.generation, 2); assert_eq!(driver.generations("generation-recovery"), vec![2]); From cf29ac331e8a75a6a94922d57225d21f41f16eb8 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 13:22:16 +0800 Subject: [PATCH 13/32] test: cover durable exec fault boundaries --- tests/general_runtime.rs | 197 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 193 insertions(+), 4 deletions(-) diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 642e348..4931762 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -60,7 +60,11 @@ struct TestDriver { stop_calls: AtomicUsize, remove_calls: AtomicUsize, exec_calls: AtomicUsize, + exec_effects: Mutex>, + exec_stdout_bytes: AtomicUsize, + exec_truncated: AtomicBool, fail_next_apply: AtomicBool, + fail_next_exec_after_effect: AtomicBool, missing_on_inspect: AtomicBool, substitute_identity: AtomicBool, unordered_logs: AtomicBool, @@ -85,7 +89,11 @@ impl TestDriver { stop_calls: AtomicUsize::new(0), remove_calls: AtomicUsize::new(0), exec_calls: AtomicUsize::new(0), + exec_effects: Mutex::new(BTreeMap::new()), + exec_stdout_bytes: AtomicUsize::new(0), + exec_truncated: AtomicBool::new(false), fail_next_apply: AtomicBool::new(false), + fail_next_exec_after_effect: AtomicBool::new(false), missing_on_inspect: AtomicBool::new(false), substitute_identity: AtomicBool::new(false), unordered_logs: AtomicBool::new(false), @@ -108,6 +116,10 @@ impl TestDriver { ManagedRuntimeClient::with_clock(store.clone(), self.clone(), Arc::new(FixedClock)); (client, store) } + + fn exec_effect_count(&self) -> usize { + self.exec_effects.lock().unwrap().len() + } } struct GenerationDriver { @@ -486,19 +498,44 @@ impl RuntimeDriver for TestDriver { if self.hang_exec.load(Ordering::SeqCst) { return std::future::pending::>().await; } + let effect_key = format!( + "{}/{}/{}", + request.unit_id, request.generation, request.request_id + ); + if let Some(result) = self.exec_effects.lock().unwrap().get(&effect_key).cloned() { + return Ok(result); + } let mut observation = unit.observation.clone(); if self.substitute_exec_spec.load(Ordering::SeqCst) { observation.spec_digest = digest('f'); } - Ok(RuntimeExecResult { + let stdout_bytes = self.exec_stdout_bytes.load(Ordering::SeqCst); + let result = RuntimeExecResult { schema: RuntimeExecResult::SCHEMA.into(), request_id: request.request_id.clone(), observation, exit_code: 0, - stdout: "ok\n".into(), + stdout: if stdout_bytes == 0 { + "ok\n".into() + } else { + "x".repeat(stdout_bytes) + }, stderr: String::new(), - truncated: false, - }) + truncated: self.exec_truncated.load(Ordering::SeqCst), + }; + self.exec_effects + .lock() + .unwrap() + .insert(effect_key, result.clone()); + if self + .fail_next_exec_after_effect + .swap(false, Ordering::SeqCst) + { + return Err(RuntimeError::Transport( + "ambiguous exec acknowledgement".into(), + )); + } + Ok(result) } } @@ -1126,6 +1163,158 @@ async fn completed_exec_replays_durably_after_client_restart_and_rejects_conflic assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); } +#[tokio::test] +async fn pending_exec_reattaches_after_an_ambiguous_after_effect_error() { + let driver = Arc::new(TestDriver::new()); + let (client, store) = driver.client(); + client + .apply(&apply( + "exec-ambiguous-apply", + service("exec-ambiguous", 1), + )) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-ambiguous-request".into(), + unit_id: "exec-ambiguous".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }; + driver + .fail_next_exec_after_effect + .store(true, Ordering::SeqCst); + + assert!(matches!( + client.exec(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("ambiguous exec") + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + assert_eq!(driver.exec_effect_count(), 1); + assert_eq!( + store + .load_request("exec-ambiguous", "exec-ambiguous-request") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); + + let restarted = + ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), Arc::new(FixedClock)); + let recovered = restarted.exec(&request).await.unwrap(); + assert_eq!(recovered.stdout, "ok\n"); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 2); + assert_eq!(driver.exec_effect_count(), 1); + assert_eq!( + store + .load_request("exec-ambiguous", "exec-ambiguous-request") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Completed + ); + assert_eq!(restarted.exec(&request).await.unwrap(), recovered); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 2); + assert_eq!(driver.exec_effect_count(), 1); +} + +#[tokio::test] +async fn cancelled_exec_releases_the_lease_and_remains_replayable() { + let driver = Arc::new(TestDriver::new()); + let (client, store) = driver.client(); + let client = Arc::new(client); + client + .apply(&apply( + "exec-cancel-apply", + service("exec-cancel", 1), + )) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-cancel-request".into(), + unit_id: "exec-cancel".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 10_000, + deadline_at_ms: None, + }; + driver.hang_exec.store(true, Ordering::SeqCst); + let operation = { + let client = client.clone(); + let request = request.clone(); + tokio::spawn(async move { client.exec(&request).await }) + }; + tokio::time::timeout(Duration::from_secs(1), async { + while driver.exec_calls.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("exec did not reach the provider before cancellation"); + operation.abort(); + assert!(operation.await.unwrap_err().is_cancelled()); + assert_eq!(driver.exec_effect_count(), 0); + assert_eq!( + store + .load_request("exec-cancel", "exec-cancel-request") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); + + driver.hang_exec.store(false, Ordering::SeqCst); + let recovered = tokio::time::timeout(Duration::from_secs(1), client.exec(&request)) + .await + .expect("cancelled exec retained its operation lease") + .unwrap(); + assert_eq!(recovered.stdout, "ok\n"); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 2); + assert_eq!(driver.exec_effect_count(), 1); +} + +#[tokio::test] +async fn maximum_exec_output_and_truncation_are_durable() { + let driver = Arc::new(TestDriver::new()); + driver + .exec_stdout_bytes + .store(16 * 1024 * 1024, Ordering::SeqCst); + driver.exec_truncated.store(true, Ordering::SeqCst); + let (client, store) = driver.client(); + client + .apply(&apply("exec-large-apply", service("exec-large", 1))) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-large-request".into(), + unit_id: "exec-large".into(), + generation: 1, + command: vec!["/bin/large-output".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }; + let result = client.exec(&request).await.unwrap(); + assert_eq!(result.stdout.len(), 16 * 1024 * 1024); + assert!(result.truncated); + result.validate().unwrap(); + assert_eq!(driver.exec_effect_count(), 1); + + let restarted = + ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + assert_eq!(restarted.exec(&request).await.unwrap(), result); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + assert_eq!(driver.exec_effect_count(), 1); + + let mut oversized = result; + oversized.stdout.push('x'); + assert!(oversized.validate().is_err()); +} + #[tokio::test] async fn completed_mutations_replay_after_deadline_provider_loss_and_removal() { let directory = tempfile::tempdir().unwrap(); From 18373b5d34063f68884aedfbbea76e2bf19a0050 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 13:23:02 +0800 Subject: [PATCH 14/32] style: format exec fault tests --- tests/general_runtime.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 4931762..2d0f270 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -1168,10 +1168,7 @@ async fn pending_exec_reattaches_after_an_ambiguous_after_effect_error() { let driver = Arc::new(TestDriver::new()); let (client, store) = driver.client(); client - .apply(&apply( - "exec-ambiguous-apply", - service("exec-ambiguous", 1), - )) + .apply(&apply("exec-ambiguous-apply", service("exec-ambiguous", 1))) .await .unwrap(); let request = RuntimeExecRequest { @@ -1227,10 +1224,7 @@ async fn cancelled_exec_releases_the_lease_and_remains_replayable() { let (client, store) = driver.client(); let client = Arc::new(client); client - .apply(&apply( - "exec-cancel-apply", - service("exec-cancel", 1), - )) + .apply(&apply("exec-cancel-apply", service("exec-cancel", 1))) .await .unwrap(); let request = RuntimeExecRequest { @@ -1304,8 +1298,7 @@ async fn maximum_exec_output_and_truncation_are_durable() { result.validate().unwrap(); assert_eq!(driver.exec_effect_count(), 1); - let restarted = - ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + let restarted = ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); assert_eq!(restarted.exec(&request).await.unwrap(), result); assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); assert_eq!(driver.exec_effect_count(), 1); From 740e718fe5587cd678dedfc5e9bf7c3b15ca438f Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 13:36:24 +0800 Subject: [PATCH 15/32] fix: preserve pending exec deadlines --- ...mplete-protocol-and-operation-semantics.md | 15 ++- src/managed.rs | 48 +++++++--- src/state/file.rs | 9 +- src/state/record.rs | 29 +++++- tests/general_runtime.rs | 93 +++++++++++++++++++ ...eceipt-v1.json => request-receipt-v2.json} | 3 +- tests/protocol_golden.rs | 2 +- 7 files changed, 176 insertions(+), 23 deletions(-) rename tests/golden/{request-receipt-v1.json => request-receipt-v2.json} (93%) diff --git a/docs/adr/0002-complete-protocol-and-operation-semantics.md b/docs/adr/0002-complete-protocol-and-operation-semantics.md index 32a15fa..3998d93 100644 --- a/docs/adr/0002-complete-protocol-and-operation-semantics.md +++ b/docs/adr/0002-complete-protocol-and-operation-semantics.md @@ -50,11 +50,13 @@ encoded explicitly as `null` in v2 and requires no ephemeral-storage capability. Adding `size_bytes` to output artifacts changes observations to `a3s.runtime.observation.v2`. Adding typed provider identity and output capability changes capabilities to `a3s.runtime.capabilities.v3`. The durable -request-journal layout uses `a3s.runtime.unit-record.v2`. +request-journal layout uses `a3s.runtime.unit-record.v2`. Persisting the +effective request deadline changes request receipts to +`a3s.runtime.request-receipt.v2`. -The Runtime core does not silently reinterpret v1 records. A caller that needs -old records owns an explicit archival decoder or migration before starting the -new client. +The Runtime core does not silently reinterpret legacy records or receipts. A +caller that needs old state owns an explicit archival decoder or migration +before starting the new client. ### 2. Provider identity is one typed value @@ -206,6 +208,11 @@ returning. Deadlines constrain unfinished work; they do not invalidate an already committed response. A pending replay remains subject to its original deadline and is never redispatched after that deadline expires. +The request receipt stores the effective absolute deadline captured on first +reservation. For Exec this is the smaller of the first attempt's relative +timeout and optional absolute deadline; a retry cannot restart that relative +timeout window. + Drivers may enforce a shorter provider-specific timeout. They must never extend the caller deadline. Exec uses the smaller of its relative `timeout_ms` and an optional absolute request deadline. diff --git a/src/managed.rs b/src/managed.rs index 9e0ac30..11eeac4 100644 --- a/src/managed.rs +++ b/src/managed.rs @@ -90,14 +90,14 @@ impl ManagedRuntimeClient { })? } - fn exec_deadline(&self, request: &RuntimeExecRequest) -> u64 { - let relative = self.clock.now_ms().saturating_add(request.timeout_ms); + fn exec_deadline(&self, request: &RuntimeExecRequest, started_at_ms: u64) -> u64 { + let relative = started_at_ms.saturating_add(request.timeout_ms); request .deadline_at_ms .map_or(relative, |absolute| absolute.min(relative)) } - async fn completed_replay( + async fn matching_receipt( &self, unit_id: &str, generation: u64, @@ -124,7 +124,21 @@ impl ManagedRuntimeClient { request_id: request_id.into(), }); } - Ok((receipt.state == RuntimeRequestState::Completed).then_some(receipt)) + Ok(Some(receipt)) + } + + async fn completed_replay( + &self, + unit_id: &str, + generation: u64, + request_id: &str, + kind: RuntimeRequestKind, + request_digest: &str, + ) -> RuntimeResult> { + Ok(self + .matching_receipt(unit_id, generation, request_id, kind, request_digest) + .await? + .filter(|receipt| receipt.state == RuntimeRequestState::Completed)) } } @@ -470,21 +484,24 @@ impl RuntimeClient for ManagedRuntimeClient { async fn exec(&self, request: &RuntimeExecRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; let request_digest = request.digest().map_err(RuntimeError::InvalidRequest)?; - if self - .completed_replay( + let operation_started_at_ms = self.clock.now_ms(); + let existing = self + .matching_receipt( &request.unit_id, request.generation, &request.request_id, RuntimeRequestKind::Exec, &request_digest, ) - .await? - .is_some() + .await?; + if existing + .as_ref() + .is_some_and(|receipt| receipt.state == RuntimeRequestState::Completed) { let _lease = self.state.acquire_operation_lease(&request.unit_id).await?; let reservation = self .state - .reserve_exec(request, self.clock.now_ms()) + .reserve_exec(request, operation_started_at_ms) .await?; if reservation.dispatch { return Err(RuntimeError::Protocol( @@ -495,7 +512,11 @@ impl RuntimeClient for ManagedRuntimeClient { RuntimeError::Protocol("completed exec receipt has no result".into()) }); } - let deadline_at_ms = Some(self.exec_deadline(request)); + let deadline_at_ms = existing + .as_ref() + .and_then(|receipt| receipt.deadline_at_ms) + .unwrap_or_else(|| self.exec_deadline(request, operation_started_at_ms)); + let deadline_at_ms = Some(deadline_at_ms); self.check_deadline(deadline_at_ms)?; let capabilities = self .bounded( @@ -519,13 +540,18 @@ impl RuntimeClient for ManagedRuntimeClient { self.check_deadline(deadline_at_ms)?; let reservation = self .state - .reserve_exec(request, self.clock.now_ms()) + .reserve_exec(request, operation_started_at_ms) .await?; if !reservation.dispatch { return reservation.receipt.exec_result.ok_or_else(|| { RuntimeError::Protocol("completed exec receipt has no result".into()) }); } + let deadline_at_ms = reservation.receipt.deadline_at_ms.ok_or_else(|| { + RuntimeError::Protocol("pending exec receipt has no effective deadline".into()) + })?; + let deadline_at_ms = Some(deadline_at_ms); + self.check_deadline(deadline_at_ms)?; let result = self .bounded( deadline_at_ms, diff --git a/src/state/file.rs b/src/state/file.rs index c535a2a..8f23bfb 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -215,6 +215,7 @@ impl FileRuntimeStateStore { fn reserve_exec_sync( &self, request: RuntimeExecRequest, + started_at_ms: u64, ) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; let _lock = self.lock(&request.unit_id)?; @@ -252,8 +253,8 @@ impl FileRuntimeStateStore { request.unit_id, record.observation.state ))); } - let receipt = - RuntimeRequestReceipt::pending_exec(&request).map_err(RuntimeError::Protocol)?; + let receipt = RuntimeRequestReceipt::pending_exec(&request, started_at_ms) + .map_err(RuntimeError::Protocol)?; receipt.validate().map_err(RuntimeError::Protocol)?; receipt_path = self.request_path(&request.unit_id, &request.request_id, true)?; atomic_write(&receipt_path, &receipt, "request receipt")?; @@ -513,11 +514,11 @@ impl RuntimeStateStore for FileRuntimeStateStore { async fn reserve_exec( &self, request: &RuntimeExecRequest, - _now_ms: u64, + now_ms: u64, ) -> RuntimeResult { let store = self.clone(); let request = request.clone(); - tokio::task::spawn_blocking(move || store.reserve_exec_sync(request)) + tokio::task::spawn_blocking(move || store.reserve_exec_sync(request, now_ms)) .await .map_err(task_error)? } diff --git a/src/state/record.rs b/src/state/record.rs index e18f15c..3661a97 100644 --- a/src/state/record.rs +++ b/src/state/record.rs @@ -45,6 +45,12 @@ pub struct RuntimeRequestReceipt { pub generation: u64, pub kind: RuntimeRequestKind, pub request_digest: String, + /// Effective absolute deadline captured when the request was first reserved. + /// + /// Exec always has a deadline because its relative timeout is converted to + /// an absolute value. Persisting it prevents a pending replay from extending + /// the original execution budget. + pub deadline_at_ms: Option, pub state: RuntimeRequestState, pub observation: Option, pub removal: Option, @@ -52,7 +58,7 @@ pub struct RuntimeRequestReceipt { } impl RuntimeRequestReceipt { - pub const SCHEMA: &'static str = "a3s.runtime.request-receipt.v1"; + pub const SCHEMA: &'static str = "a3s.runtime.request-receipt.v2"; pub(crate) fn pending_apply(request: &RuntimeApplyRequest) -> Result { Ok(Self::pending( @@ -61,6 +67,7 @@ impl RuntimeRequestReceipt { request.spec.generation, RuntimeRequestKind::Apply, request.digest()?, + request.deadline_at_ms, )) } @@ -74,16 +81,27 @@ impl RuntimeRequestReceipt { request.generation, kind.into(), request.digest()?, + request.deadline_at_ms, )) } - pub(crate) fn pending_exec(request: &RuntimeExecRequest) -> Result { + pub(crate) fn pending_exec( + request: &RuntimeExecRequest, + started_at_ms: u64, + ) -> Result { + let relative_deadline = started_at_ms.saturating_add(request.timeout_ms); + let deadline_at_ms = request + .deadline_at_ms + .map_or(relative_deadline, |absolute| { + absolute.min(relative_deadline) + }); Ok(Self::pending( request.request_id.clone(), request.unit_id.clone(), request.generation, RuntimeRequestKind::Exec, request.digest()?, + Some(deadline_at_ms), )) } @@ -93,6 +111,7 @@ impl RuntimeRequestReceipt { generation: u64, kind: RuntimeRequestKind, request_digest: String, + deadline_at_ms: Option, ) -> Self { Self { schema: Self::SCHEMA.into(), @@ -101,6 +120,7 @@ impl RuntimeRequestReceipt { generation, kind, request_digest, + deadline_at_ms, state: RuntimeRequestState::Pending, observation: None, removal: None, @@ -141,6 +161,11 @@ impl RuntimeRequestReceipt { if self.generation == 0 { return Err("Runtime request receipt generation must be positive".into()); } + if self.deadline_at_ms == Some(0) + || (self.kind == RuntimeRequestKind::Exec && self.deadline_at_ms.is_none()) + { + return Err("Runtime request receipt deadline is invalid".into()); + } crate::contract::validate_digest(&self.request_digest)?; match ( self.kind, diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 2d0f270..574b044 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -1308,6 +1308,44 @@ async fn maximum_exec_output_and_truncation_are_durable() { assert!(oversized.validate().is_err()); } +#[tokio::test] +async fn lc_exec_002_oversized_provider_result_stays_pending() { + let driver = Arc::new(TestDriver::new()); + driver + .exec_stdout_bytes + .store(16 * 1024 * 1024 + 1, Ordering::SeqCst); + let (client, store) = driver.client(); + client + .apply(&apply("exec-oversized-apply", service("exec-oversized", 1))) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-oversized-request".into(), + unit_id: "exec-oversized".into(), + generation: 1, + command: vec!["/bin/oversized-output".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }; + + for expected_calls in [1, 2] { + assert!(matches!( + client.exec(&request).await, + Err(RuntimeError::Protocol(message)) + if message.contains("exec output exceeds protocol limits") + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), expected_calls); + assert_eq!(driver.exec_effect_count(), 1); + let receipt = store + .load_request("exec-oversized", "exec-oversized-request") + .await + .unwrap(); + assert_eq!(receipt.state, a3s_runtime::RuntimeRequestState::Pending); + assert!(receipt.exec_result.is_none()); + } +} + #[tokio::test] async fn completed_mutations_replay_after_deadline_provider_loss_and_removal() { let directory = tempfile::tempdir().unwrap(); @@ -1798,6 +1836,61 @@ async fn exec_uses_the_smaller_relative_or_absolute_deadline() { assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); } +#[tokio::test] +async fn lc_exec_001_pending_replay_cannot_extend_original_relative_deadline() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let clock = Arc::new(ManualClock::new(NOW)); + let client = ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); + client + .apply(&apply( + "exec-relative-replay-apply", + service("exec-relative-replay", 1), + )) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-relative-replay-request".into(), + unit_id: "exec-relative-replay".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 10, + deadline_at_ms: None, + }; + driver.hang_exec.store(true, Ordering::SeqCst); + + assert!(matches!( + client.exec(&request).await, + Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider exec") + )); + let pending = store + .load_request("exec-relative-replay", "exec-relative-replay-request") + .await + .unwrap(); + assert_eq!(pending.state, a3s_runtime::RuntimeRequestState::Pending); + assert_eq!(pending.deadline_at_ms, Some(NOW + 10)); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + + clock.set(NOW + 10); + driver.hang_exec.store(false, Ordering::SeqCst); + let restarted = ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock); + assert!(matches!( + restarted.exec(&request).await, + Err(RuntimeError::DeadlineExceeded(_)) + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + assert_eq!( + store + .load_request("exec-relative-replay", "exec-relative-replay-request") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); +} + #[tokio::test] async fn provider_loss_becomes_a_durable_unknown_observation() { let driver = Arc::new(TestDriver::new()); diff --git a/tests/golden/request-receipt-v1.json b/tests/golden/request-receipt-v2.json similarity index 93% rename from tests/golden/request-receipt-v1.json rename to tests/golden/request-receipt-v2.json index 7cde408..a210564 100644 --- a/tests/golden/request-receipt-v1.json +++ b/tests/golden/request-receipt-v2.json @@ -1,10 +1,11 @@ { - "schema": "a3s.runtime.request-receipt.v1", + "schema": "a3s.runtime.request-receipt.v2", "request_id": "golden-exec", "unit_id": "golden-service", "generation": 3, "kind": "exec", "request_digest": "sha256:861f6cad44261c49a89b5cd12920ca3977d833b5c9143d748db945a08939bad0", + "deadline_at_ms": 200000, "state": "completed", "observation": null, "removal": null, diff --git a/tests/protocol_golden.rs b/tests/protocol_golden.rs index 2c92fc6..f1140bc 100644 --- a/tests/protocol_golden.rs +++ b/tests/protocol_golden.rs @@ -150,7 +150,7 @@ fn ct_schema_001_every_top_level_wire_record_has_a_versioned_golden_fixture() { RuntimeUnitRecord::validate, ); assert_top_level_fixture::( - include_str!("golden/request-receipt-v1.json"), + include_str!("golden/request-receipt-v2.json"), RuntimeRequestReceipt::SCHEMA, RuntimeRequestReceipt::validate, ); From 56cb6c90e2def69f8e4582e6edd0b3d7f25fb4d0 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 13:50:08 +0800 Subject: [PATCH 16/32] test: kill receipt-first completion processes --- src/state/file.rs | 14 ++ src/state/file/tests.rs | 365 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 379 insertions(+) create mode 100644 src/state/file/tests.rs diff --git a/src/state/file.rs b/src/state/file.rs index 8f23bfb..a44f0c0 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -329,6 +329,8 @@ impl FileRuntimeStateStore { receipt.complete_with_observation(observation.clone()); receipt.validate().map_err(RuntimeError::Protocol)?; atomic_write(&receipt_path, &receipt, "request receipt")?; + #[cfg(test)] + test_failpoint("state.complete-observation.after-receipt-publish"); } } @@ -367,6 +369,8 @@ impl FileRuntimeStateStore { receipt.complete_with_removal(removal.clone()); receipt.validate().map_err(RuntimeError::Protocol)?; atomic_write(&receipt_path, &receipt, "request receipt")?; + #[cfg(test)] + test_failpoint("state.complete-removal.after-receipt-publish"); } record.removed_at_ms = Some(removal.removed_at_ms); record.validate().map_err(RuntimeError::Protocol)?; @@ -407,6 +411,8 @@ impl FileRuntimeStateStore { receipt.complete_with_exec_result(result.clone()); receipt.validate().map_err(RuntimeError::Protocol)?; atomic_write(&receipt_path, &receipt, "request receipt")?; + #[cfg(test)] + test_failpoint("state.complete-exec.after-receipt-publish"); } record.observation = result.observation; record.validate().map_err(RuntimeError::Protocol)?; @@ -1023,3 +1029,11 @@ fn task_error(error: tokio::task::JoinError) -> RuntimeError { fn io_error(action: &'static str) -> impl FnOnce(std::io::Error) -> RuntimeError { move |error| RuntimeError::Transport(format!("could not {action}: {error}")) } + +#[cfg(test)] +fn test_failpoint(name: &str) { + tests::hit_failpoint(name); +} + +#[cfg(test)] +mod tests; diff --git a/src/state/file/tests.rs b/src/state/file/tests.rs new file mode 100644 index 0000000..d654e13 --- /dev/null +++ b/src/state/file/tests.rs @@ -0,0 +1,365 @@ +use super::*; +use crate::contract::{ + ArtifactRef, IsolationLevel, NetworkMode, ResourceLimits, RestartPolicy, RuntimeActionRequest, + RuntimeApplyRequest, RuntimeExecRequest, RuntimeExecResult, RuntimeNetworkSpec, + RuntimeObservation, RuntimeProcessSpec, RuntimeRemoval, RuntimeUnitClass, RuntimeUnitSpec, + RuntimeUnitState, +}; +use crate::RuntimeRequestState; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; +use std::thread; +use std::time::{Duration, Instant}; + +const NOW: u64 = 10_000; +const UNIT_ID: &str = "receipt-first-unit"; +const APPLY_ID: &str = "receipt-first-apply"; +const STOP_ID: &str = "receipt-first-stop"; +const REMOVE_ID: &str = "receipt-first-remove"; +const EXEC_ID: &str = "receipt-first-exec"; +const FAILPOINT_ENV: &str = "A3S_RUNTIME_TEST_FAILPOINT"; +const READY_ENV: &str = "A3S_RUNTIME_TEST_FAILPOINT_READY"; +const ROOT_ENV: &str = "A3S_RUNTIME_TEST_STATE_ROOT"; +const OPERATION_ENV: &str = "A3S_RUNTIME_TEST_COMPLETION_KIND"; +const OBSERVATION_FAILPOINT: &str = "state.complete-observation.after-receipt-publish"; +const REMOVAL_FAILPOINT: &str = "state.complete-removal.after-receipt-publish"; +const EXEC_FAILPOINT: &str = "state.complete-exec.after-receipt-publish"; + +pub(super) fn hit_failpoint(name: &str) { + if !matches!(std::env::var(FAILPOINT_ENV), Ok(value) if value == name) { + return; + } + let ready = std::env::var(READY_ENV).expect("test failpoint ready path"); + std::fs::write(ready, name).expect("publish test failpoint readiness"); + loop { + thread::park_timeout(Duration::from_secs(60)); + } +} + +fn digest(character: char) -> String { + format!("sha256:{}", character.to_string().repeat(64)) +} + +fn spec() -> RuntimeUnitSpec { + RuntimeUnitSpec { + schema: RuntimeUnitSpec::SCHEMA.into(), + unit_id: UNIT_ID.into(), + generation: 1, + class: RuntimeUnitClass::Service, + artifact: ArtifactRef { + uri: format!( + "oci://registry.example/a3s/runtime@sha256:{}", + "a".repeat(64) + ), + digest: digest('a'), + media_type: "application/vnd.oci.image.manifest.v1+json".into(), + }, + process: RuntimeProcessSpec { + command: vec!["/bin/service".into()], + args: Vec::new(), + working_directory: None, + environment: BTreeMap::new(), + }, + mounts: Vec::new(), + secrets: Vec::new(), + network: RuntimeNetworkSpec { + mode: NetworkMode::None, + ports: Vec::new(), + }, + resources: ResourceLimits { + cpu_millis: 100, + memory_bytes: 64 * 1024 * 1024, + pids: 32, + ephemeral_storage_bytes: None, + execution_timeout_ms: None, + }, + isolation: IsolationLevel::Container, + health: None, + restart: RestartPolicy::Always, + outputs: Vec::new(), + semantics_profile_digest: None, + } +} + +fn apply_request() -> RuntimeApplyRequest { + RuntimeApplyRequest { + schema: RuntimeApplyRequest::SCHEMA.into(), + request_id: APPLY_ID.into(), + deadline_at_ms: Some(NOW + 60_000), + spec: spec(), + } +} + +fn action(request_id: &str) -> RuntimeActionRequest { + RuntimeActionRequest { + schema: RuntimeActionRequest::SCHEMA.into(), + request_id: request_id.into(), + unit_id: UNIT_ID.into(), + generation: 1, + deadline_at_ms: Some(NOW + 60_000), + } +} + +fn exec_request() -> RuntimeExecRequest { + RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: EXEC_ID.into(), + unit_id: UNIT_ID.into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: Some(NOW + 60_000), + } +} + +fn provider_observation( + mut observation: RuntimeObservation, + state: RuntimeUnitState, + observed_at_ms: u64, +) -> RuntimeObservation { + observation.state = state; + observation.provider_resource_id = Some(format!("provider/{UNIT_ID}/1")); + observation.provider_build = Some("receipt-first-driver/1".into()); + observation.observed_at_ms = observed_at_ms; + observation.started_at_ms = Some(NOW); + observation.finished_at_ms = state.is_terminal().then_some(observed_at_ms); + observation.health = None; + observation.outputs.clear(); + observation.failure = None; + observation +} + +fn prepare_running(store: &FileRuntimeStateStore) -> RuntimeObservation { + let request = apply_request(); + let reservation = store + .reserve_apply_sync(request.clone(), NOW) + .expect("reserve initial apply"); + let running = provider_observation( + reservation.record.observation, + RuntimeUnitState::Running, + NOW + 1, + ); + store + .update_observation_sync(Some(request.request_id), running.clone()) + .expect("complete initial apply"); + running +} + +fn spawn_completion_helper(root: &Path, ready: &Path, operation: &str, failpoint: &str) -> Child { + Command::new(std::env::current_exe().expect("current test executable")) + .arg("subprocess_receipt_first_completion_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env(ROOT_ENV, root) + .env(READY_ENV, ready) + .env(OPERATION_ENV, operation) + .env(FAILPOINT_ENV, failpoint) + .spawn() + .expect("spawn receipt-first completion helper") +} + +fn kill_at_failpoint(child: &mut Child, ready: &Path, case_id: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + while !ready.is_file() { + if let Some(status) = child.try_wait().expect("inspect completion helper") { + panic!("{case_id}: helper exited before failpoint: {status}"); + } + assert!( + Instant::now() < deadline, + "{case_id}: helper did not reach failpoint" + ); + thread::sleep(Duration::from_millis(10)); + } + child.kill().expect("kill completion helper"); + let status = child.wait().expect("wait for killed completion helper"); + assert!(!status.success(), "{case_id}: helper was not killed"); +} + +fn run_receipt_first_crash( + operation: &str, + failpoint: &str, + prepare: impl FnOnce(&FileRuntimeStateStore), + verify: impl FnOnce(&FileRuntimeStateStore), +) { + let directory = tempfile::tempdir().expect("receipt-first state root"); + let store = FileRuntimeStateStore::new(directory.path()); + prepare(&store); + let ready = directory.path().join(format!("{operation}.ready")); + let mut child = spawn_completion_helper(directory.path(), &ready, operation, failpoint); + kill_at_failpoint(&mut child, &ready, operation); + verify(&FileRuntimeStateStore::new(directory.path())); +} + +#[test] +fn subprocess_receipt_first_completion_helper() { + let Ok(root) = std::env::var(ROOT_ENV) else { + return; + }; + let operation = std::env::var(OPERATION_ENV).expect("completion kind"); + let store = FileRuntimeStateStore::new(PathBuf::from(root)); + let record = store.load_sync(UNIT_ID).expect("load helper unit"); + match operation.as_str() { + "crash-apply-receipt-first" => { + let running = + provider_observation(record.observation, RuntimeUnitState::Running, NOW + 1); + store + .update_observation_sync(Some(APPLY_ID.into()), running) + .expect("complete apply"); + } + "crash-stop-receipt-first" => { + let stopped = + provider_observation(record.observation, RuntimeUnitState::Stopped, NOW + 2); + store + .update_observation_sync(Some(STOP_ID.into()), stopped) + .expect("complete stop"); + } + "crash-remove-receipt-first" => { + store + .complete_removal_sync(RuntimeRemoval { + schema: RuntimeRemoval::SCHEMA.into(), + request_id: REMOVE_ID.into(), + unit_id: UNIT_ID.into(), + generation: 1, + removed_at_ms: NOW + 3, + already_absent: false, + }) + .expect("complete removal"); + } + "crash-exec-receipt-first" => { + let mut refreshed = record.observation; + refreshed.observed_at_ms += 1; + store + .complete_exec_sync(RuntimeExecResult { + schema: RuntimeExecResult::SCHEMA.into(), + request_id: EXEC_ID.into(), + observation: refreshed, + exit_code: 0, + stdout: "ok\n".into(), + stderr: String::new(), + truncated: false, + }) + .expect("complete exec"); + } + other => panic!("unknown receipt-first completion kind {other:?}"), + } +} + +#[test] +fn state_crash_001_receipt_first_process_kills_reconcile_every_result_kind() { + run_receipt_first_crash( + "crash-apply-receipt-first", + OBSERVATION_FAILPOINT, + |store| { + store + .reserve_apply_sync(apply_request(), NOW) + .expect("reserve apply"); + }, + |store| { + assert_eq!( + store + .load_sync(UNIT_ID) + .expect("pre-replay unit") + .observation + .state, + RuntimeUnitState::Accepted + ); + assert_eq!( + store + .load_request_sync(UNIT_ID, APPLY_ID) + .expect("completed apply receipt") + .state, + RuntimeRequestState::Completed + ); + let replay = store + .reserve_apply_sync(apply_request(), NOW + 2) + .expect("replay apply"); + assert!(!replay.dispatch); + assert_eq!(replay.record.observation.state, RuntimeUnitState::Running); + assert_eq!( + store + .load_sync(UNIT_ID) + .expect("reconciled unit") + .observation, + replay.record.observation + ); + }, + ); + + run_receipt_first_crash( + "crash-stop-receipt-first", + OBSERVATION_FAILPOINT, + |store| { + prepare_running(store); + store + .reserve_action_sync(RuntimeActionKind::Stop, action(STOP_ID), NOW + 1) + .expect("reserve stop"); + }, + |store| { + assert_eq!( + store + .load_sync(UNIT_ID) + .expect("pre-replay unit") + .observation + .state, + RuntimeUnitState::Running + ); + let replay = store + .reserve_action_sync(RuntimeActionKind::Stop, action(STOP_ID), NOW + 3) + .expect("replay stop"); + assert!(!replay.dispatch); + assert_eq!(replay.record.observation.state, RuntimeUnitState::Stopped); + }, + ); + + run_receipt_first_crash( + "crash-remove-receipt-first", + REMOVAL_FAILPOINT, + |store| { + prepare_running(store); + store + .reserve_action_sync(RuntimeActionKind::Remove, action(REMOVE_ID), NOW + 1) + .expect("reserve removal"); + }, + |store| { + assert!(store + .load_sync(UNIT_ID) + .expect("pre-replay unit") + .removed_at_ms + .is_none()); + let replay = store + .reserve_action_sync(RuntimeActionKind::Remove, action(REMOVE_ID), NOW + 4) + .expect("replay removal"); + assert!(!replay.dispatch); + assert_eq!(replay.record.removed_at_ms, Some(NOW + 3)); + }, + ); + + run_receipt_first_crash( + "crash-exec-receipt-first", + EXEC_FAILPOINT, + |store| { + prepare_running(store); + store + .reserve_exec_sync(exec_request(), NOW + 1) + .expect("reserve exec"); + }, + |store| { + let before = store + .load_sync(UNIT_ID) + .expect("pre-replay unit") + .observation + .observed_at_ms; + let replay = store + .reserve_exec_sync(exec_request(), NOW + 2) + .expect("replay exec"); + assert!(!replay.dispatch); + assert!(replay.record.observation.observed_at_ms > before); + assert_eq!( + replay.receipt.state, + RuntimeRequestState::Completed, + "exec result must remain durable after process kill" + ); + }, + ); +} From 897f97d6b9f1840572babe38f578442bfe616a25 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 13:58:01 +0800 Subject: [PATCH 17/32] test: generate complete state transition matrix --- ...mplete-protocol-and-operation-semantics.md | 5 + src/state/file.rs | 8 +- src/state/mod.rs | 3 + src/state/transition_tests.rs | 225 ++++++++++++++++++ 4 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 src/state/transition_tests.rs diff --git a/docs/adr/0002-complete-protocol-and-operation-semantics.md b/docs/adr/0002-complete-protocol-and-operation-semantics.md index 3998d93..0fa6185 100644 --- a/docs/adr/0002-complete-protocol-and-operation-semantics.md +++ b/docs/adr/0002-complete-protocol-and-operation-semantics.md @@ -112,6 +112,11 @@ core durably records `unknown`. A later same-generation apply may adopt one new provider resource ID. Once that observation is completed, exact replay returns the replacement without another provider create. +`unknown` is a confirmed-loss state, not a second `accepted` state. Recovery +may move from `unknown` to a valid provider-backed intermediate or operation +result and may adopt a replacement provider identity, but it cannot regress to +`accepted`. + ### 6. Operation postconditions are explicit Apply returns a provider-backed observation that has advanced beyond diff --git a/src/state/file.rs b/src/state/file.rs index a44f0c0..4915fca 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -734,7 +734,6 @@ pub(crate) fn validate_transition( return Ok(()); } let allowed = current.state == next.state - || current.state == RuntimeUnitState::Unknown || matches!( (current.state, next.state), (RuntimeUnitState::Accepted, RuntimeUnitState::Preparing) @@ -765,6 +764,13 @@ pub(crate) fn validate_transition( | (RuntimeUnitState::Stopping, RuntimeUnitState::Stopped) | (RuntimeUnitState::Stopping, RuntimeUnitState::Failed) | (RuntimeUnitState::Stopping, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Preparing) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Starting) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Running) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Stopping) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Failed) ); if !allowed { return Err(RuntimeError::Protocol(format!( diff --git a/src/state/mod.rs b/src/state/mod.rs index e8df649..13071f1 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -14,6 +14,9 @@ pub use record::{ RuntimeStateReservation, RuntimeUnitRecord, }; +#[cfg(test)] +mod transition_tests; + /// Owned guard for one unit's cross-process operation lease. Implementations /// release the lease when the guard is dropped. pub trait RuntimeOperationLease: Send {} diff --git a/src/state/transition_tests.rs b/src/state/transition_tests.rs new file mode 100644 index 0000000..3a6f107 --- /dev/null +++ b/src/state/transition_tests.rs @@ -0,0 +1,225 @@ +use super::file::validate_transition; +use crate::contract::{ + ArtifactRef, IsolationLevel, NetworkMode, ResourceLimits, RestartPolicy, RuntimeFailure, + RuntimeHealthObservation, RuntimeHealthState, RuntimeNetworkSpec, RuntimeObservation, + RuntimeProcessSpec, RuntimeUnitClass, RuntimeUnitSpec, RuntimeUnitState, +}; +use std::collections::BTreeMap; + +const STATES: [RuntimeUnitState; 9] = [ + RuntimeUnitState::Accepted, + RuntimeUnitState::Preparing, + RuntimeUnitState::Starting, + RuntimeUnitState::Running, + RuntimeUnitState::Stopping, + RuntimeUnitState::Stopped, + RuntimeUnitState::Succeeded, + RuntimeUnitState::Failed, + RuntimeUnitState::Unknown, +]; + +fn digest(character: char) -> String { + format!("sha256:{}", character.to_string().repeat(64)) +} + +fn spec(class: RuntimeUnitClass) -> RuntimeUnitSpec { + let class_name = match class { + RuntimeUnitClass::Task => "task", + RuntimeUnitClass::Service => "service", + }; + RuntimeUnitSpec { + schema: RuntimeUnitSpec::SCHEMA.into(), + unit_id: format!("transition-{class_name}"), + generation: 1, + class, + artifact: ArtifactRef { + uri: format!( + "oci://registry.example/a3s/transition@sha256:{}", + "a".repeat(64) + ), + digest: digest('a'), + media_type: "application/vnd.oci.image.manifest.v1+json".into(), + }, + process: RuntimeProcessSpec { + command: vec!["/bin/workload".into()], + args: Vec::new(), + working_directory: None, + environment: BTreeMap::new(), + }, + mounts: Vec::new(), + secrets: Vec::new(), + network: RuntimeNetworkSpec { + mode: NetworkMode::None, + ports: Vec::new(), + }, + resources: ResourceLimits { + cpu_millis: 100, + memory_bytes: 64 * 1024 * 1024, + pids: 32, + ephemeral_storage_bytes: None, + execution_timeout_ms: (class == RuntimeUnitClass::Task).then_some(60_000), + }, + isolation: IsolationLevel::Container, + health: None, + restart: match class { + RuntimeUnitClass::Task => RestartPolicy::Never, + RuntimeUnitClass::Service => RestartPolicy::Always, + }, + outputs: Vec::new(), + semantics_profile_digest: None, + } +} + +fn observation( + spec: &RuntimeUnitSpec, + state: RuntimeUnitState, + observed_at_ms: u64, +) -> RuntimeObservation { + let provider_backed = state != RuntimeUnitState::Accepted; + RuntimeObservation { + schema: RuntimeObservation::SCHEMA.into(), + unit_id: spec.unit_id.clone(), + generation: spec.generation, + spec_digest: spec.digest().expect("valid transition specification"), + class: spec.class, + state, + provider_resource_id: provider_backed.then(|| "provider/transition/1".into()), + provider_build: provider_backed.then(|| "transition-driver/1".into()), + observed_at_ms, + started_at_ms: provider_backed.then_some(100), + finished_at_ms: state.is_terminal().then_some(observed_at_ms), + health: None, + outputs: Vec::new(), + usage: None, + evidence: None, + provider_attestation: None, + failure: (state == RuntimeUnitState::Failed).then(|| RuntimeFailure { + code: "provider_failed".into(), + message: "injected transition failure".into(), + retryable: false, + }), + } +} + +fn valid_for_class(class: RuntimeUnitClass, state: RuntimeUnitState) -> bool { + class != RuntimeUnitClass::Service || state != RuntimeUnitState::Succeeded +} + +fn expected_transition(from: RuntimeUnitState, to: RuntimeUnitState) -> bool { + if from.is_terminal() { + return false; + } + from == to + || matches!( + (from, to), + (RuntimeUnitState::Accepted, RuntimeUnitState::Preparing) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Starting) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Running) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Failed) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Starting) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Running) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Stopping) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Failed) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Starting, RuntimeUnitState::Running) + | (RuntimeUnitState::Starting, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Starting, RuntimeUnitState::Stopping) + | (RuntimeUnitState::Starting, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Starting, RuntimeUnitState::Failed) + | (RuntimeUnitState::Starting, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Running, RuntimeUnitState::Stopping) + | (RuntimeUnitState::Running, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Running, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Running, RuntimeUnitState::Failed) + | (RuntimeUnitState::Running, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Stopping, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Stopping, RuntimeUnitState::Failed) + | (RuntimeUnitState::Stopping, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Preparing) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Starting) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Running) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Stopping) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Failed) + ) +} + +#[test] +fn state_transition_001_task_and_service_matrices_have_complete_oracles() { + for class in [RuntimeUnitClass::Task, RuntimeUnitClass::Service] { + let spec = spec(class); + for from in STATES + .into_iter() + .filter(|state| valid_for_class(class, *state)) + { + for to in STATES + .into_iter() + .filter(|state| valid_for_class(class, *state)) + { + let current = observation(&spec, from, 200); + let next = observation(&spec, to, 201); + let actual = validate_transition(¤t, &next, &spec).is_ok(); + assert_eq!( + actual, + expected_transition(from, to), + "STATE-TRANSITION-{class:?}-{from:?}-{to:?}" + ); + } + } + } +} + +#[test] +fn state_transition_002_terminal_identity_time_and_class_rules_fail_closed() { + for class in [RuntimeUnitClass::Task, RuntimeUnitClass::Service] { + let spec = spec(class); + for terminal in [ + RuntimeUnitState::Stopped, + RuntimeUnitState::Succeeded, + RuntimeUnitState::Failed, + ] + .into_iter() + .filter(|state| valid_for_class(class, *state)) + { + let current = observation(&spec, terminal, 200); + assert!( + validate_transition(¤t, ¤t, &spec).is_ok(), + "STATE-TERMINAL-REPLAY-{class:?}-{terminal:?}" + ); + } + } + + let task = spec(RuntimeUnitClass::Task); + let running = observation(&task, RuntimeUnitState::Running, 200); + let mut substituted = observation(&task, RuntimeUnitState::Running, 201); + substituted.provider_resource_id = Some("provider/substituted/1".into()); + assert!(validate_transition(&running, &substituted, &task).is_err()); + + let unknown = observation(&task, RuntimeUnitState::Unknown, 200); + assert!(validate_transition(&unknown, &substituted, &task).is_ok()); + let accepted = observation(&task, RuntimeUnitState::Accepted, 201); + assert!(validate_transition(&unknown, &accepted, &task).is_err()); + + let mut backwards = observation(&task, RuntimeUnitState::Running, 199); + backwards.provider_resource_id = running.provider_resource_id.clone(); + assert!(validate_transition(&running, &backwards, &task).is_err()); + + let service = spec(RuntimeUnitClass::Service); + let service_running = observation(&service, RuntimeUnitState::Running, 200); + let service_succeeded = observation(&service, RuntimeUnitState::Succeeded, 201); + assert!(validate_transition(&service_running, &service_succeeded, &service).is_err()); + + let mut task_with_health = observation(&task, RuntimeUnitState::Running, 201); + task_with_health.health = Some(RuntimeHealthObservation { + state: RuntimeHealthState::Healthy, + checked_at_ms: 201, + message: None, + }); + assert!(validate_transition(&running, &task_with_health, &task).is_err()); +} From 086667fa57a43b07e7592f94b2c3f534d9c7f65a Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 14:05:34 +0800 Subject: [PATCH 18/32] test: harden hostile state file boundaries --- src/state/file.rs | 2 + src/state/file/tests.rs | 111 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/src/state/file.rs b/src/state/file.rs index 4915fca..5c15b43 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -850,6 +850,8 @@ fn owner_only_open(path: &Path, label: &str) -> RuntimeResult { file.set_permissions(std::fs::Permissions::from_mode(0o600)) .map_err(io_error("secure state lock"))?; } + let metadata = file.metadata().map_err(io_error("inspect state lock"))?; + verify_owner_only_file(&metadata, path, label)?; Ok(file) } diff --git a/src/state/file/tests.rs b/src/state/file/tests.rs index d654e13..4f4a8b4 100644 --- a/src/state/file/tests.rs +++ b/src/state/file/tests.rs @@ -363,3 +363,114 @@ fn state_crash_001_receipt_first_process_kills_reconcile_every_result_kind() { }, ); } + +#[cfg(unix)] +#[test] +fn state_fs_001_hard_linked_lock_record_and_receipt_files_fail_closed() { + let directory = tempfile::tempdir().expect("hard-link state root"); + let store = FileRuntimeStateStore::new(directory.path()); + store + .reserve_apply_sync(apply_request(), NOW) + .expect("reserve hard-link fixture"); + let key = storage_key(UNIT_ID); + + let state_lock = directory.path().join("locks").join(format!("{key}.lock")); + let state_lock_alias = directory.path().join("state-lock.alias"); + std::fs::hard_link(&state_lock, &state_lock_alias).expect("hard-link state lock"); + assert!(matches!( + store.load_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); + std::fs::remove_file(state_lock_alias).expect("remove state-lock alias"); + + let record = store.record_path(UNIT_ID, false).expect("record path"); + let record_alias = directory.path().join("record.alias"); + std::fs::hard_link(&record, &record_alias).expect("hard-link record"); + assert!(matches!( + store.load_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); + std::fs::remove_file(record_alias).expect("remove record alias"); + + let receipt = store + .request_path(UNIT_ID, APPLY_ID, false) + .expect("receipt path"); + let receipt_alias = directory.path().join("receipt.alias"); + std::fs::hard_link(&receipt, &receipt_alias).expect("hard-link receipt"); + assert!(matches!( + store.load_request_sync(UNIT_ID, APPLY_ID), + Err(RuntimeError::Protocol(_)) + )); + std::fs::remove_file(receipt_alias).expect("remove receipt alias"); + + drop( + store + .acquire_operation_lease_sync(UNIT_ID) + .expect("create operation lock"), + ); + let operation_lock = directory + .path() + .join("operations") + .join(format!("{key}.lock")); + let operation_alias = directory.path().join("operation-lock.alias"); + std::fs::hard_link(&operation_lock, &operation_alias).expect("hard-link operation lock"); + assert!(matches!( + store.acquire_operation_lease_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); +} + +#[cfg(unix)] +#[test] +fn state_fs_002_corrupt_permission_and_non_regular_boundaries_fail_closed() { + use std::os::unix::fs::PermissionsExt; + use std::os::unix::net::UnixListener; + + let truncated = tempfile::tempdir().expect("truncated state root"); + let store = FileRuntimeStateStore::new(truncated.path()); + store + .reserve_apply_sync(apply_request(), NOW) + .expect("reserve truncated fixture"); + let record = store.record_path(UNIT_ID, false).expect("record path"); + std::fs::write(&record, b"{").expect("truncate record"); + assert!(matches!( + store.load_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); + + let invalid_utf8 = tempfile::tempdir().expect("invalid UTF-8 state root"); + let store = FileRuntimeStateStore::new(invalid_utf8.path()); + store + .reserve_apply_sync(apply_request(), NOW) + .expect("reserve invalid UTF-8 fixture"); + let receipt = store + .request_path(UNIT_ID, APPLY_ID, false) + .expect("receipt path"); + std::fs::write(&receipt, [0xff]).expect("corrupt receipt encoding"); + assert!(matches!( + store.load_request_sync(UNIT_ID, APPLY_ID), + Err(RuntimeError::Protocol(_)) + )); + + let permissions = tempfile::tempdir().expect("permission state root"); + let store = FileRuntimeStateStore::new(permissions.path()); + store + .reserve_apply_sync(apply_request(), NOW) + .expect("reserve permission fixture"); + let record = store.record_path(UNIT_ID, false).expect("record path"); + std::fs::set_permissions(&record, std::fs::Permissions::from_mode(0o644)) + .expect("relax record permissions"); + assert!(matches!( + store.load_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); + + let non_regular = tempfile::tempdir().expect("non-regular state root"); + let store = FileRuntimeStateStore::new(non_regular.path()); + let record = store.record_path(UNIT_ID, true).expect("record path"); + let _socket = UnixListener::bind(&record).expect("bind record socket"); + assert!(matches!( + store.load_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); +} From 52d80c9e716cb3d965cc8f168670619df0b3d762 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 14:17:00 +0800 Subject: [PATCH 19/32] test: add deterministic provider fault matrix --- tests/deterministic_faults.rs | 363 ++++++++++++++++++++++++++++ tests/support/fault_driver.rs | 436 ++++++++++++++++++++++++++++++++++ tests/support/fixtures.rs | 76 ++++++ tests/support/mod.rs | 2 + 4 files changed, 877 insertions(+) create mode 100644 tests/deterministic_faults.rs create mode 100644 tests/support/fault_driver.rs create mode 100644 tests/support/fixtures.rs create mode 100644 tests/support/mod.rs diff --git a/tests/deterministic_faults.rs b/tests/deterministic_faults.rs new file mode 100644 index 0000000..ec63e97 --- /dev/null +++ b/tests/deterministic_faults.rs @@ -0,0 +1,363 @@ +mod support; + +use a3s_runtime::contract::{RuntimeInspection, RuntimeLogQuery}; +use a3s_runtime::{ + FileRuntimeStateStore, ManagedRuntimeClient, RuntimeClient, RuntimeError, RuntimeRequestState, + RuntimeStateStore, +}; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use support::fault_driver::{DeterministicFaultDriver, FaultBoundary, FaultMode, FaultOperation}; +use support::fixtures::{action_request, apply_request, exec_request}; + +fn client( + directory: &tempfile::TempDir, + driver: Arc, +) -> (ManagedRuntimeClient, Arc) { + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + (ManagedRuntimeClient::new(store.clone(), driver), store) +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock after Unix epoch") + .as_millis() as u64 +} + +fn one_if(condition: bool) -> usize { + if condition { + 1 + } else { + 0 + } +} + +#[tokio::test] +async fn fault_apply_001_transport_boundaries_replay_without_duplicate_resources() { + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + let directory = tempfile::tempdir().expect("fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = format!("fault-apply-{boundary:?}").to_ascii_lowercase(); + let request = apply_request("fault-apply-request", &unit_id); + driver.arm(FaultOperation::Apply, boundary, FaultMode::Transport); + + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Apply") + )); + assert_eq!( + driver.resource_count(&unit_id), + one_if(boundary == FaultBoundary::AfterEffect) + ); + assert_eq!( + store + .load_request(&unit_id, &request.request_id) + .await + .expect("pending apply receipt") + .state, + RuntimeRequestState::Pending + ); + + let restarted = ManagedRuntimeClient::new(store.clone(), driver.clone()); + let recovered = restarted.apply(&request).await.expect("replay apply"); + let expected_resource_id = format!("provider/{unit_id}/g1"); + assert_eq!( + recovered.provider_resource_id.as_deref(), + Some(expected_resource_id.as_str()) + ); + assert_eq!(driver.resource_count(&unit_id), 1); + assert_eq!(driver.effect_count(FaultOperation::Apply), 1); + assert_eq!( + store + .load_request(&unit_id, &request.request_id) + .await + .expect("completed apply receipt") + .state, + RuntimeRequestState::Completed + ); + assert!(driver + .trace() + .iter() + .any(|event| event.operation == FaultOperation::Apply && event.boundary == boundary)); + } +} + +#[tokio::test] +async fn fault_mutation_002_stop_remove_and_exec_replay_each_provider_boundary() { + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + let directory = tempfile::tempdir().expect("stop fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = format!("fault-stop-{boundary:?}").to_ascii_lowercase(); + client + .apply(&apply_request("fault-stop-apply", &unit_id)) + .await + .expect("prepare running stop fixture"); + let request = action_request("fault-stop-request", &unit_id); + driver.arm(FaultOperation::Stop, boundary, FaultMode::Transport); + + assert!(matches!( + client.stop(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Stop") + )); + let expected_after_fault = match boundary { + FaultBoundary::BeforeEffect => a3s_runtime::contract::RuntimeUnitState::Running, + FaultBoundary::AfterEffect => a3s_runtime::contract::RuntimeUnitState::Stopped, + }; + assert_eq!( + driver.resource_state(&unit_id, 1), + Some(expected_after_fault) + ); + let restarted = ManagedRuntimeClient::new(store.clone(), driver.clone()); + let RuntimeInspection::Found { observation, .. } = + restarted.stop(&request).await.expect("replay stop") + else { + panic!("replayed stop must preserve the unit"); + }; + assert_eq!( + observation.state, + a3s_runtime::contract::RuntimeUnitState::Stopped + ); + assert_eq!(driver.effect_count(FaultOperation::Stop), 1); + assert_eq!( + store + .load_request(&unit_id, &request.request_id) + .await + .expect("completed stop receipt") + .state, + RuntimeRequestState::Completed + ); + } + + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + let directory = tempfile::tempdir().expect("remove fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = format!("fault-remove-{boundary:?}").to_ascii_lowercase(); + client + .apply(&apply_request("fault-remove-apply", &unit_id)) + .await + .expect("prepare running remove fixture"); + let request = action_request("fault-remove-request", &unit_id); + driver.arm(FaultOperation::Remove, boundary, FaultMode::Transport); + + assert!(matches!( + client.remove(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Remove") + )); + assert_eq!( + driver.resource_count(&unit_id), + one_if(boundary == FaultBoundary::BeforeEffect) + ); + let restarted = ManagedRuntimeClient::new(store.clone(), driver.clone()); + restarted.remove(&request).await.expect("replay removal"); + assert_eq!(driver.resource_count(&unit_id), 0); + assert_eq!(driver.effect_count(FaultOperation::Remove), 1); + assert_eq!( + store + .load_request(&unit_id, &request.request_id) + .await + .expect("completed remove receipt") + .state, + RuntimeRequestState::Completed + ); + } + + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + let directory = tempfile::tempdir().expect("exec fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = format!("fault-exec-{boundary:?}").to_ascii_lowercase(); + client + .apply(&apply_request("fault-exec-apply", &unit_id)) + .await + .expect("prepare running exec fixture"); + let request = exec_request("fault-exec-request", &unit_id); + driver.arm(FaultOperation::Exec, boundary, FaultMode::Transport); + + assert!(matches!( + client.exec(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Exec") + )); + assert_eq!( + driver.effect_count(FaultOperation::Exec), + one_if(boundary == FaultBoundary::AfterEffect) + ); + assert_eq!( + store + .load_request(&unit_id, &request.request_id) + .await + .expect("pending exec receipt") + .state, + RuntimeRequestState::Pending + ); + let restarted = ManagedRuntimeClient::new(store.clone(), driver.clone()); + let result = restarted.exec(&request).await.expect("replay exec"); + assert_eq!(result.stdout, "ok\n"); + assert_eq!(driver.effect_count(FaultOperation::Exec), 1); + assert_eq!( + restarted.exec(&request).await.expect("exact exec replay"), + result + ); + assert_eq!(driver.effect_count(FaultOperation::Exec), 1); + } +} + +#[tokio::test] +async fn fault_apply_003_timeout_after_effect_needs_a_new_request_to_reconcile() { + let directory = tempfile::tempdir().expect("timeout fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = "fault-apply-timeout"; + let mut request = apply_request("fault-timeout-request", unit_id); + request.deadline_at_ms = Some(now_ms() + 200); + driver.arm( + FaultOperation::Apply, + FaultBoundary::AfterEffect, + FaultMode::Hang, + ); + + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider apply") + )); + assert_eq!(driver.resource_count(unit_id), 1); + assert_eq!(driver.effect_count(FaultOperation::Apply), 1); + assert_eq!( + store + .load_request(unit_id, &request.request_id) + .await + .expect("timed-out pending receipt") + .state, + RuntimeRequestState::Pending + ); + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(_)) + )); + assert_eq!(driver.effect_count(FaultOperation::Apply), 1); + + let replacement_request = apply_request("fault-timeout-reconcile", unit_id); + let recovered = client + .apply(&replacement_request) + .await + .expect("new request reattaches timed-out provider effect"); + assert_eq!( + recovered.state, + a3s_runtime::contract::RuntimeUnitState::Running + ); + assert_eq!(driver.resource_count(unit_id), 1); + assert_eq!(driver.effect_count(FaultOperation::Apply), 1); +} + +#[tokio::test] +async fn fault_apply_004_provider_panic_releases_the_operation_lease() { + let directory = tempfile::tempdir().expect("panic fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let client = Arc::new(client); + let unit_id = "fault-apply-panic"; + let request = apply_request("fault-panic-request", unit_id); + driver.arm( + FaultOperation::Apply, + FaultBoundary::AfterEffect, + FaultMode::Panic, + ); + + let operation = { + let client = client.clone(); + let request = request.clone(); + tokio::spawn(async move { client.apply(&request).await }) + }; + assert!(operation + .await + .expect_err("provider panic must unwind") + .is_panic()); + assert_eq!(driver.resource_count(unit_id), 1); + assert_eq!( + store + .load_request(unit_id, &request.request_id) + .await + .expect("panic pending receipt") + .state, + RuntimeRequestState::Pending + ); + + let restarted = ManagedRuntimeClient::new(store, driver.clone()); + let recovered = tokio::time::timeout(Duration::from_secs(1), restarted.apply(&request)) + .await + .expect("panic retained operation lease") + .expect("replay apply after provider panic"); + assert_eq!( + recovered.state, + a3s_runtime::contract::RuntimeUnitState::Running + ); + assert_eq!(driver.effect_count(FaultOperation::Apply), 1); +} + +#[tokio::test] +async fn fault_read_005_capabilities_inspect_and_logs_fail_without_state_mutation() { + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + let directory = tempfile::tempdir().expect("capability fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = format!("fault-capability-{boundary:?}").to_ascii_lowercase(); + driver.arm(FaultOperation::Capabilities, boundary, FaultMode::Transport); + assert!(matches!( + client.apply(&apply_request("fault-capability-request", &unit_id)).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Capabilities") + )); + assert!(matches!( + store.load(&unit_id).await, + Err(RuntimeError::NotFound { .. }) + )); + assert_eq!(driver.resource_count(&unit_id), 0); + } + + let directory = tempfile::tempdir().expect("read fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = "fault-read-unit"; + client + .apply(&apply_request("fault-read-apply", unit_id)) + .await + .expect("prepare read fault fixture"); + let expected = store.load(unit_id).await.expect("durable read fixture"); + + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + driver.arm(FaultOperation::Inspect, boundary, FaultMode::Transport); + assert!(matches!( + client.inspect(unit_id).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Inspect") + )); + assert_eq!( + store + .load(unit_id) + .await + .expect("state after inspect fault"), + expected + ); + } + + let query = RuntimeLogQuery { + schema: RuntimeLogQuery::SCHEMA.into(), + unit_id: unit_id.into(), + generation: 1, + cursor: None, + limit: 10, + stream: None, + }; + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + driver.arm(FaultOperation::Logs, boundary, FaultMode::Transport); + assert!(matches!( + client.logs(&query).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Logs") + )); + assert_eq!( + store.load(unit_id).await.expect("state after log fault"), + expected + ); + } +} diff --git a/tests/support/fault_driver.rs b/tests/support/fault_driver.rs new file mode 100644 index 0000000..c1563b6 --- /dev/null +++ b/tests/support/fault_driver.rs @@ -0,0 +1,436 @@ +use a3s_runtime::contract::{ + IsolationLevel, NetworkMode, ResourceControl, RuntimeActionRequest, RuntimeCapabilities, + RuntimeExecRequest, RuntimeExecResult, RuntimeInspection, RuntimeLogChunk, RuntimeLogQuery, + RuntimeLogStream, RuntimeObservation, RuntimeRemoval, RuntimeUnitClass, RuntimeUnitSpec, + RuntimeUnitState, +}; +use a3s_runtime::{ + ProviderId, RuntimeDriver, RuntimeError, RuntimeFeature, RuntimeResult, RuntimeUnitRecord, +}; +use async_trait::async_trait; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Mutex; + +pub const IMAGE_MEDIA_TYPE: &str = "application/vnd.oci.image.manifest.v1+json"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum FaultOperation { + Capabilities, + Apply, + Inspect, + Stop, + Remove, + Logs, + Exec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FaultBoundary { + BeforeEffect, + AfterEffect, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FaultMode { + Transport, + Hang, + Panic, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ArmedFault { + operation: FaultOperation, + boundary: FaultBoundary, + mode: FaultMode, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TraceEvent { + pub operation: FaultOperation, + pub boundary: FaultBoundary, +} + +#[derive(Debug, Clone)] +struct ProviderResource { + observation: RuntimeObservation, +} + +pub struct DeterministicFaultDriver { + provider_id: ProviderId, + fault: Mutex>, + trace: Mutex>, + resources: Mutex>>, + exec_results: Mutex>, + effects: Mutex>, +} + +impl Default for DeterministicFaultDriver { + fn default() -> Self { + Self::new() + } +} + +impl DeterministicFaultDriver { + pub fn new() -> Self { + Self { + provider_id: ProviderId::parse("deterministic-fault-runtime") + .expect("valid deterministic provider ID"), + fault: Mutex::new(None), + trace: Mutex::new(Vec::new()), + resources: Mutex::new(BTreeMap::new()), + exec_results: Mutex::new(BTreeMap::new()), + effects: Mutex::new(BTreeMap::new()), + } + } + + pub fn arm(&self, operation: FaultOperation, boundary: FaultBoundary, mode: FaultMode) { + let mut fault = self.fault.lock().expect("fault plan lock"); + assert!(fault.is_none(), "only one deterministic fault may be armed"); + *fault = Some(ArmedFault { + operation, + boundary, + mode, + }); + } + + pub fn trace(&self) -> Vec { + self.trace.lock().expect("trace lock").clone() + } + + pub fn resource_count(&self, unit_id: &str) -> usize { + self.resources + .lock() + .expect("provider resource lock") + .get(unit_id) + .map(BTreeMap::len) + .unwrap_or_default() + } + + pub fn resource_state(&self, unit_id: &str, generation: u64) -> Option { + self.resources + .lock() + .expect("provider resource lock") + .get(unit_id) + .and_then(|generations| generations.get(&generation)) + .map(|resource| resource.observation.state) + } + + pub fn effect_count(&self, operation: FaultOperation) -> usize { + self.effects + .lock() + .expect("effect counter lock") + .get(&operation) + .copied() + .unwrap_or_default() + } + + async fn boundary( + &self, + operation: FaultOperation, + boundary: FaultBoundary, + ) -> RuntimeResult<()> { + self.trace.lock().expect("trace lock").push(TraceEvent { + operation, + boundary, + }); + let mode = { + let mut armed = self.fault.lock().expect("fault plan lock"); + if armed + .as_ref() + .is_some_and(|fault| fault.operation == operation && fault.boundary == boundary) + { + armed.take().map(|fault| fault.mode) + } else { + None + } + }; + match mode { + None => Ok(()), + Some(FaultMode::Transport) => Err(RuntimeError::Transport(format!( + "injected {operation:?} {boundary:?} transport fault" + ))), + Some(FaultMode::Hang) => std::future::pending::>().await, + Some(FaultMode::Panic) => { + panic!("injected {operation:?} {boundary:?} provider panic") + } + } + } + + fn record_effect(&self, operation: FaultOperation) { + *self + .effects + .lock() + .expect("effect counter lock") + .entry(operation) + .or_default() += 1; + } + + fn capabilities_value(&self) -> RuntimeCapabilities { + RuntimeCapabilities { + schema: RuntimeCapabilities::SCHEMA.into(), + provider_id: self.provider_id.clone(), + provider_build: "deterministic-fault-driver/1".into(), + unit_classes: vec![RuntimeUnitClass::Task, RuntimeUnitClass::Service], + artifact_media_types: vec![IMAGE_MEDIA_TYPE.into()], + isolation_levels: vec![IsolationLevel::Container], + network_modes: vec![NetworkMode::None], + mount_kinds: Vec::new(), + health_check_kinds: Vec::new(), + resource_controls: vec![ + ResourceControl::Cpu, + ResourceControl::Memory, + ResourceControl::Pids, + ResourceControl::ExecutionTimeout, + ], + features: vec![ + RuntimeFeature::DurableIdentity, + RuntimeFeature::Stop, + RuntimeFeature::Remove, + RuntimeFeature::Logs, + RuntimeFeature::Exec, + ], + } + } + + fn provider_observation( + &self, + spec: &RuntimeUnitSpec, + current: &RuntimeObservation, + resource_id: String, + ) -> RuntimeObservation { + let mut observation = current.clone(); + observation.state = match spec.class { + RuntimeUnitClass::Task => RuntimeUnitState::Succeeded, + RuntimeUnitClass::Service => RuntimeUnitState::Running, + }; + observation.provider_resource_id = Some(resource_id); + observation.provider_build = Some("deterministic-fault-driver/1".into()); + observation.observed_at_ms = current.observed_at_ms.saturating_add(1); + observation.started_at_ms = Some(current.observed_at_ms); + observation.finished_at_ms = observation + .state + .is_terminal() + .then_some(observation.observed_at_ms); + observation.health = None; + observation.outputs.clear(); + observation.failure = None; + observation + } + + fn exec_key(request: &RuntimeExecRequest) -> String { + format!( + "{}/{}/{}", + request.unit_id, request.generation, request.request_id + ) + } +} + +#[async_trait] +impl RuntimeDriver for DeterministicFaultDriver { + fn provider_id(&self) -> &ProviderId { + &self.provider_id + } + + async fn capabilities(&self) -> RuntimeResult { + self.boundary(FaultOperation::Capabilities, FaultBoundary::BeforeEffect) + .await?; + let capabilities = self.capabilities_value(); + self.boundary(FaultOperation::Capabilities, FaultBoundary::AfterEffect) + .await?; + Ok(capabilities) + } + + async fn apply( + &self, + spec: &RuntimeUnitSpec, + current: &RuntimeObservation, + ) -> RuntimeResult { + self.boundary(FaultOperation::Apply, FaultBoundary::BeforeEffect) + .await?; + let (observation, created) = { + let mut resources = self.resources.lock().expect("provider resource lock"); + let generations = resources.entry(spec.unit_id.clone()).or_default(); + let mut created = false; + let observation = if let Some(resource) = generations.get(&spec.generation) { + resource.observation.clone() + } else { + created = true; + self.provider_observation( + spec, + current, + format!("provider/{}/g{}", spec.unit_id, spec.generation), + ) + }; + generations.insert( + spec.generation, + ProviderResource { + observation: observation.clone(), + }, + ); + let stale = generations + .keys() + .copied() + .filter(|generation| *generation != spec.generation) + .collect::>(); + for generation in stale { + generations.remove(&generation); + } + (observation, created) + }; + if created { + self.record_effect(FaultOperation::Apply); + } + self.boundary(FaultOperation::Apply, FaultBoundary::AfterEffect) + .await?; + Ok(observation) + } + + async fn inspect(&self, unit: &RuntimeUnitRecord) -> RuntimeResult { + self.boundary(FaultOperation::Inspect, FaultBoundary::BeforeEffect) + .await?; + let inspection = self + .resources + .lock() + .expect("provider resource lock") + .get(&unit.spec.unit_id) + .and_then(|generations| generations.get(&unit.spec.generation)) + .map_or_else( + || RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), + unit_id: unit.spec.unit_id.clone(), + last_generation: Some(unit.spec.generation), + }, + |resource| RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), + observation: Box::new(resource.observation.clone()), + }, + ); + self.boundary(FaultOperation::Inspect, FaultBoundary::AfterEffect) + .await?; + Ok(inspection) + } + + async fn stop( + &self, + unit: &RuntimeUnitRecord, + _request: &RuntimeActionRequest, + ) -> RuntimeResult { + self.boundary(FaultOperation::Stop, FaultBoundary::BeforeEffect) + .await?; + let (observation, changed) = { + let mut resources = self.resources.lock().expect("provider resource lock"); + let resource = resources + .get_mut(&unit.spec.unit_id) + .and_then(|generations| generations.get_mut(&unit.spec.generation)) + .ok_or_else(|| RuntimeError::NotFound { + unit_id: unit.spec.unit_id.clone(), + })?; + let changed = resource.observation.state != RuntimeUnitState::Stopped; + if changed { + resource.observation.state = RuntimeUnitState::Stopped; + resource.observation.observed_at_ms = + resource.observation.observed_at_ms.saturating_add(1); + resource.observation.finished_at_ms = Some(resource.observation.observed_at_ms); + resource.observation.health = None; + } + (resource.observation.clone(), changed) + }; + if changed { + self.record_effect(FaultOperation::Stop); + } + self.boundary(FaultOperation::Stop, FaultBoundary::AfterEffect) + .await?; + Ok(observation) + } + + async fn remove( + &self, + unit: &RuntimeUnitRecord, + request: &RuntimeActionRequest, + ) -> RuntimeResult { + self.boundary(FaultOperation::Remove, FaultBoundary::BeforeEffect) + .await?; + let existed = { + let mut resources = self.resources.lock().expect("provider resource lock"); + let removed = resources + .get_mut(&unit.spec.unit_id) + .and_then(|generations| generations.remove(&unit.spec.generation)) + .is_some(); + if resources + .get(&unit.spec.unit_id) + .is_some_and(BTreeMap::is_empty) + { + resources.remove(&unit.spec.unit_id); + } + removed + }; + if existed { + self.record_effect(FaultOperation::Remove); + } + let removal = RuntimeRemoval { + schema: RuntimeRemoval::SCHEMA.into(), + request_id: request.request_id.clone(), + unit_id: unit.spec.unit_id.clone(), + generation: unit.spec.generation, + removed_at_ms: unit.observation.observed_at_ms.saturating_add(1), + already_absent: !existed, + }; + self.boundary(FaultOperation::Remove, FaultBoundary::AfterEffect) + .await?; + Ok(removal) + } + + async fn logs( + &self, + unit: &RuntimeUnitRecord, + _query: &RuntimeLogQuery, + ) -> RuntimeResult> { + self.boundary(FaultOperation::Logs, FaultBoundary::BeforeEffect) + .await?; + let chunks = vec![RuntimeLogChunk { + schema: RuntimeLogChunk::SCHEMA.into(), + cursor: "fault-cursor-1".into(), + sequence: 1, + observed_at_ms: unit.observation.observed_at_ms, + stream: RuntimeLogStream::Stdout, + data: "ready\n".into(), + }]; + self.boundary(FaultOperation::Logs, FaultBoundary::AfterEffect) + .await?; + Ok(chunks) + } + + async fn exec( + &self, + unit: &RuntimeUnitRecord, + request: &RuntimeExecRequest, + ) -> RuntimeResult { + self.boundary(FaultOperation::Exec, FaultBoundary::BeforeEffect) + .await?; + let key = Self::exec_key(request); + let (result, created) = { + let mut results = self.exec_results.lock().expect("exec result lock"); + if let Some(result) = results.get(&key) { + (result.clone(), false) + } else { + let result = RuntimeExecResult { + schema: RuntimeExecResult::SCHEMA.into(), + request_id: request.request_id.clone(), + observation: unit.observation.clone(), + exit_code: 0, + stdout: "ok\n".into(), + stderr: String::new(), + truncated: false, + }; + results.insert(key, result.clone()); + (result, true) + } + }; + if created { + self.record_effect(FaultOperation::Exec); + } + self.boundary(FaultOperation::Exec, FaultBoundary::AfterEffect) + .await?; + Ok(result) + } +} diff --git a/tests/support/fixtures.rs b/tests/support/fixtures.rs new file mode 100644 index 0000000..f54cb5e --- /dev/null +++ b/tests/support/fixtures.rs @@ -0,0 +1,76 @@ +use super::fault_driver::IMAGE_MEDIA_TYPE; +use a3s_runtime::contract::{ + ArtifactRef, IsolationLevel, NetworkMode, ResourceLimits, RestartPolicy, RuntimeActionRequest, + RuntimeApplyRequest, RuntimeExecRequest, RuntimeNetworkSpec, RuntimeProcessSpec, + RuntimeUnitClass, RuntimeUnitSpec, +}; +use std::collections::BTreeMap; + +pub fn service_spec(unit_id: &str) -> RuntimeUnitSpec { + RuntimeUnitSpec { + schema: RuntimeUnitSpec::SCHEMA.into(), + unit_id: unit_id.into(), + generation: 1, + class: RuntimeUnitClass::Service, + artifact: ArtifactRef { + uri: format!("oci://registry.example/a3s/fault@sha256:{}", "a".repeat(64)), + digest: format!("sha256:{}", "a".repeat(64)), + media_type: IMAGE_MEDIA_TYPE.into(), + }, + process: RuntimeProcessSpec { + command: vec!["/bin/service".into()], + args: Vec::new(), + working_directory: None, + environment: BTreeMap::new(), + }, + mounts: Vec::new(), + secrets: Vec::new(), + network: RuntimeNetworkSpec { + mode: NetworkMode::None, + ports: Vec::new(), + }, + resources: ResourceLimits { + cpu_millis: 100, + memory_bytes: 64 * 1024 * 1024, + pids: 32, + ephemeral_storage_bytes: None, + execution_timeout_ms: None, + }, + isolation: IsolationLevel::Container, + health: None, + restart: RestartPolicy::Always, + outputs: Vec::new(), + semantics_profile_digest: None, + } +} + +pub fn apply_request(request_id: &str, unit_id: &str) -> RuntimeApplyRequest { + RuntimeApplyRequest { + schema: RuntimeApplyRequest::SCHEMA.into(), + request_id: request_id.into(), + deadline_at_ms: None, + spec: service_spec(unit_id), + } +} + +pub fn action_request(request_id: &str, unit_id: &str) -> RuntimeActionRequest { + RuntimeActionRequest { + schema: RuntimeActionRequest::SCHEMA.into(), + request_id: request_id.into(), + unit_id: unit_id.into(), + generation: 1, + deadline_at_ms: None, + } +} + +pub fn exec_request(request_id: &str, unit_id: &str) -> RuntimeExecRequest { + RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: request_id.into(), + unit_id: unit_id.into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + } +} diff --git a/tests/support/mod.rs b/tests/support/mod.rs new file mode 100644 index 0000000..333d99a --- /dev/null +++ b/tests/support/mod.rs @@ -0,0 +1,2 @@ +pub mod fault_driver; +pub mod fixtures; From c2b5bae07dd30a29cca55f2bd79da9292a5dad8a Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 14:18:09 +0800 Subject: [PATCH 20/32] test: import fault feature from contract --- tests/support/fault_driver.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/support/fault_driver.rs b/tests/support/fault_driver.rs index c1563b6..431b3a5 100644 --- a/tests/support/fault_driver.rs +++ b/tests/support/fault_driver.rs @@ -1,12 +1,10 @@ use a3s_runtime::contract::{ IsolationLevel, NetworkMode, ResourceControl, RuntimeActionRequest, RuntimeCapabilities, - RuntimeExecRequest, RuntimeExecResult, RuntimeInspection, RuntimeLogChunk, RuntimeLogQuery, - RuntimeLogStream, RuntimeObservation, RuntimeRemoval, RuntimeUnitClass, RuntimeUnitSpec, - RuntimeUnitState, -}; -use a3s_runtime::{ - ProviderId, RuntimeDriver, RuntimeError, RuntimeFeature, RuntimeResult, RuntimeUnitRecord, + RuntimeExecRequest, RuntimeExecResult, RuntimeFeature, RuntimeInspection, RuntimeLogChunk, + RuntimeLogQuery, RuntimeLogStream, RuntimeObservation, RuntimeRemoval, RuntimeUnitClass, + RuntimeUnitSpec, RuntimeUnitState, }; +use a3s_runtime::{ProviderId, RuntimeDriver, RuntimeError, RuntimeResult, RuntimeUnitRecord}; use async_trait::async_trait; use std::collections::{BTreeMap, BTreeSet}; use std::sync::Mutex; From 37b1f2af65882b7d7abe441c74d82a8b28fe7991 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 14:21:04 +0800 Subject: [PATCH 21/32] ci: require stable and MSRV runtime gates --- .github/workflows/ci.yml | 55 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e56fc57 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,55 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: runtime-ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + +jobs: + linux: + name: Linux / Rust ${{ matrix.toolchain }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + toolchain: + - "1.85.0" + - stable + + steps: + - name: Check out exact commit + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.toolchain }} + components: clippy, rustfmt + + - name: Verify formatting + run: cargo fmt --all --check + + - name: Run unit, integration, process, and golden tests + run: cargo test --locked --all-targets + + - name: Reject Clippy warnings + run: cargo clippy --locked --all-targets -- -D warnings + + - name: Reject rustdoc warnings + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --locked --no-deps From 92f24a9ed9dd54d88d9a8335951dc1785333515d Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 14:44:42 +0800 Subject: [PATCH 22/32] fix: recover atomic state writes after crashes --- src/state/file.rs | 239 +----------------------------- src/state/file/fs.rs | 317 ++++++++++++++++++++++++++++++++++++++++ src/state/file/tests.rs | 240 ++++++++++++++++++++++++++++++ 3 files changed, 563 insertions(+), 233 deletions(-) create mode 100644 src/state/file/fs.rs diff --git a/src/state/file.rs b/src/state/file.rs index 5c15b43..b96dd26 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -1,3 +1,7 @@ +use self::fs::{ + atomic_write, ensure_directory, io_error, owner_only_open, path_exists, read_optional_receipt, + read_optional_record, read_required_receipt, read_required_record, +}; use super::{ RuntimeActionKind, RuntimeOperationLease, RuntimeRequestKind, RuntimeRequestReceipt, RuntimeRequestState, RuntimeStateReservation, RuntimeStateStore, RuntimeUnitRecord, @@ -9,15 +13,11 @@ use crate::contract::{ use crate::{RuntimeError, RuntimeResult}; use async_trait::async_trait; use fs2::FileExt; -use serde::de::DeserializeOwned; -use serde::Serialize; use sha2::{Digest, Sha256}; -use std::fs::{File, OpenOptions}; -use std::io::{Read, Write}; +use std::fs::File; use std::path::{Path, PathBuf}; -const MAX_RECORD_BYTES: u64 = 8 * 1024 * 1024; -const MAX_RECEIPT_BYTES: u64 = 40 * 1024 * 1024; +mod fs; #[derive(Debug, Clone)] pub struct FileRuntimeStateStore { @@ -812,236 +812,9 @@ fn validate_request_id(request_id: &str) -> RuntimeResult<()> { .map_err(RuntimeError::InvalidRequest) } -fn ensure_directory(path: &Path) -> RuntimeResult<()> { - if path_exists(path)? { - let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state path"))?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(RuntimeError::Protocol(format!( - "Runtime state path {} is not a real directory", - path.display() - ))); - } - verify_owner(&metadata, path, "state directory")?; - } else { - std::fs::create_dir_all(path).map_err(io_error("create state directory"))?; - } - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) - .map_err(io_error("secure state directory"))?; - } - Ok(()) -} - -fn owner_only_open(path: &Path, label: &str) -> RuntimeResult { - reject_symlink(path, label)?; - let mut options = OpenOptions::new(); - options.create(true).read(true).write(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600).custom_flags(libc::O_NOFOLLOW); - } - let file = options.open(path).map_err(io_error("open state lock"))?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - file.set_permissions(std::fs::Permissions::from_mode(0o600)) - .map_err(io_error("secure state lock"))?; - } - let metadata = file.metadata().map_err(io_error("inspect state lock"))?; - verify_owner_only_file(&metadata, path, label)?; - Ok(file) -} - -fn read_required_record(path: &Path, unit_id: &str) -> RuntimeResult { - let record = read_optional_record(path)?.ok_or_else(|| RuntimeError::NotFound { - unit_id: unit_id.into(), - })?; - if record.spec.unit_id != unit_id { - return Err(RuntimeError::Protocol("Runtime state key mismatch".into())); - } - Ok(record) -} - -fn read_optional_record(path: &Path) -> RuntimeResult> { - read_optional_json(path, MAX_RECORD_BYTES, "state record") -} - -fn read_required_receipt( - path: &Path, - unit_id: &str, - request_id: &str, -) -> RuntimeResult { - let receipt = read_optional_receipt(path)?.ok_or_else(|| RuntimeError::RequestNotFound { - unit_id: unit_id.into(), - request_id: request_id.into(), - })?; - if receipt.unit_id != unit_id || receipt.request_id != request_id { - return Err(RuntimeError::Protocol( - "Runtime request receipt storage key mismatch".into(), - )); - } - Ok(receipt) -} - -fn read_optional_receipt(path: &Path) -> RuntimeResult> { - let receipt: Option = - read_optional_json(path, MAX_RECEIPT_BYTES, "request receipt")?; - if let Some(receipt) = &receipt { - receipt.validate().map_err(RuntimeError::Protocol)?; - } - Ok(receipt) -} - -fn read_optional_json( - path: &Path, - max_bytes: u64, - label: &str, -) -> RuntimeResult> { - if !regular_file_exists(path, label)? { - return Ok(None); - } - let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state file"))?; - verify_owner_only_file(&metadata, path, label)?; - if metadata.len() > max_bytes { - return Err(RuntimeError::Protocol(format!( - "Runtime {label} exceeds its size limit" - ))); - } - let capacity = usize::try_from(metadata.len()).map_err(|_| { - RuntimeError::Protocol(format!("Runtime {label} size cannot be represented")) - })?; - let mut bytes = Vec::with_capacity(capacity); - let mut options = OpenOptions::new(); - options.read(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.custom_flags(libc::O_NOFOLLOW); - } - options - .open(path) - .and_then(|mut file| file.read_to_end(&mut bytes)) - .map_err(io_error("read state file"))?; - let value = serde_json::from_slice(&bytes) - .map_err(|error| RuntimeError::Protocol(format!("invalid {label}: {error}")))?; - Ok(Some(value)) -} - -fn regular_file_exists(path: &Path, label: &str) -> RuntimeResult { - match std::fs::symlink_metadata(path) { - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { - Err(RuntimeError::Protocol(format!( - "Runtime {label} {} is not a regular file", - path.display() - ))) - } - Ok(_) => Ok(true), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(io_error("inspect state file")(error)), - } -} - -fn path_exists(path: &Path) -> RuntimeResult { - match std::fs::symlink_metadata(path) { - Ok(_) => Ok(true), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(io_error("inspect state path")(error)), - } -} - -fn reject_symlink(path: &Path, label: &str) -> RuntimeResult<()> { - match std::fs::symlink_metadata(path) { - Ok(metadata) if metadata.file_type().is_symlink() => Err(RuntimeError::Protocol(format!( - "{label} {} must not be a symbolic link", - path.display() - ))), - Ok(_) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(io_error("inspect state file")(error)), - } -} - -#[cfg(unix)] -fn verify_owner(metadata: &std::fs::Metadata, path: &Path, label: &str) -> RuntimeResult<()> { - use std::os::unix::fs::MetadataExt; - if metadata.uid() != unsafe { libc::geteuid() } { - return Err(RuntimeError::Protocol(format!( - "Runtime {label} {} is owned by another user", - path.display() - ))); - } - Ok(()) -} - -#[cfg(not(unix))] -fn verify_owner(_metadata: &std::fs::Metadata, _path: &Path, _label: &str) -> RuntimeResult<()> { - Ok(()) -} - -fn verify_owner_only_file( - metadata: &std::fs::Metadata, - path: &Path, - label: &str, -) -> RuntimeResult<()> { - verify_owner(metadata, path, label)?; - #[cfg(unix)] - { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - if metadata.permissions().mode() & 0o077 != 0 || metadata.nlink() != 1 { - return Err(RuntimeError::Protocol(format!( - "Runtime {label} {} is not an owner-only unlinked file", - path.display() - ))); - } - } - Ok(()) -} - -fn atomic_write(path: &Path, value: &T, label: &str) -> RuntimeResult<()> { - let parent = path - .parent() - .ok_or_else(|| RuntimeError::Protocol(format!("{label} has no parent")))?; - let bytes = serde_json::to_vec(value) - .map_err(|error| RuntimeError::Protocol(format!("encode {label}: {error}")))?; - let mut temporary = - tempfile::NamedTempFile::new_in(parent).map_err(io_error("create state staging file"))?; - temporary - .write_all(&bytes) - .and_then(|()| temporary.as_file().sync_all()) - .map_err(io_error("write state file"))?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - temporary - .as_file() - .set_permissions(std::fs::Permissions::from_mode(0o600)) - .map_err(io_error("secure state file"))?; - } - temporary - .persist(path) - .map_err(|error| io_error("publish state file")(error.error))?; - #[cfg(unix)] - File::open(parent) - .and_then(|directory| directory.sync_all()) - .map_err(io_error("sync state directory"))?; - Ok(()) -} - fn task_error(error: tokio::task::JoinError) -> RuntimeError { RuntimeError::Transport(format!("Runtime state task failed: {error}")) } -fn io_error(action: &'static str) -> impl FnOnce(std::io::Error) -> RuntimeError { - move |error| RuntimeError::Transport(format!("could not {action}: {error}")) -} - -#[cfg(test)] -fn test_failpoint(name: &str) { - tests::hit_failpoint(name); -} - #[cfg(test)] mod tests; diff --git a/src/state/file/fs.rs b/src/state/file/fs.rs new file mode 100644 index 0000000..f197f05 --- /dev/null +++ b/src/state/file/fs.rs @@ -0,0 +1,317 @@ +use crate::state::{RuntimeRequestReceipt, RuntimeUnitRecord}; +use crate::{RuntimeError, RuntimeResult}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::Path; + +const MAX_RECORD_BYTES: u64 = 8 * 1024 * 1024; +const MAX_RECEIPT_BYTES: u64 = 40 * 1024 * 1024; + +pub(super) fn ensure_directory(path: &Path) -> RuntimeResult<()> { + if path_exists(path)? { + let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state path"))?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(RuntimeError::Protocol(format!( + "Runtime state path {} is not a real directory", + path.display() + ))); + } + verify_owner(&metadata, path, "state directory")?; + } else { + std::fs::create_dir_all(path).map_err(io_error("create state directory"))?; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) + .map_err(io_error("secure state directory"))?; + } + Ok(()) +} + +pub(super) fn owner_only_open(path: &Path, label: &str) -> RuntimeResult { + reject_symlink(path, label)?; + let mut options = OpenOptions::new(); + options.create(true).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let file = options.open(path).map_err(io_error("open state lock"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .map_err(io_error("secure state lock"))?; + } + let metadata = file.metadata().map_err(io_error("inspect state lock"))?; + verify_owner_only_file(&metadata, path, label)?; + Ok(file) +} + +pub(super) fn read_required_record(path: &Path, unit_id: &str) -> RuntimeResult { + let record = read_optional_record(path)?.ok_or_else(|| RuntimeError::NotFound { + unit_id: unit_id.into(), + })?; + if record.spec.unit_id != unit_id { + return Err(RuntimeError::Protocol("Runtime state key mismatch".into())); + } + Ok(record) +} + +pub(super) fn read_optional_record(path: &Path) -> RuntimeResult> { + read_optional_json(path, MAX_RECORD_BYTES, "state record") +} + +pub(super) fn read_required_receipt( + path: &Path, + unit_id: &str, + request_id: &str, +) -> RuntimeResult { + let receipt = read_optional_receipt(path)?.ok_or_else(|| RuntimeError::RequestNotFound { + unit_id: unit_id.into(), + request_id: request_id.into(), + })?; + if receipt.unit_id != unit_id || receipt.request_id != request_id { + return Err(RuntimeError::Protocol( + "Runtime request receipt storage key mismatch".into(), + )); + } + Ok(receipt) +} + +pub(super) fn read_optional_receipt(path: &Path) -> RuntimeResult> { + let receipt: Option = + read_optional_json(path, MAX_RECEIPT_BYTES, "request receipt")?; + if let Some(receipt) = &receipt { + receipt.validate().map_err(RuntimeError::Protocol)?; + } + Ok(receipt) +} + +fn read_optional_json( + path: &Path, + max_bytes: u64, + label: &str, +) -> RuntimeResult> { + if !regular_file_exists(path, label)? { + return Ok(None); + } + let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state file"))?; + verify_owner_only_file(&metadata, path, label)?; + if metadata.len() > max_bytes { + return Err(RuntimeError::Protocol(format!( + "Runtime {label} exceeds its size limit" + ))); + } + let capacity = usize::try_from(metadata.len()).map_err(|_| { + RuntimeError::Protocol(format!("Runtime {label} size cannot be represented")) + })?; + let mut bytes = Vec::with_capacity(capacity); + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + options + .open(path) + .and_then(|mut file| file.read_to_end(&mut bytes)) + .map_err(io_error("read state file"))?; + let value = serde_json::from_slice(&bytes) + .map_err(|error| RuntimeError::Protocol(format!("invalid {label}: {error}")))?; + Ok(Some(value)) +} + +fn regular_file_exists(path: &Path, label: &str) -> RuntimeResult { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + Err(RuntimeError::Protocol(format!( + "Runtime {label} {} is not a regular file", + path.display() + ))) + } + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(io_error("inspect state file")(error)), + } +} + +pub(super) fn path_exists(path: &Path) -> RuntimeResult { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(io_error("inspect state path")(error)), + } +} + +fn reject_symlink(path: &Path, label: &str) -> RuntimeResult<()> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => Err(RuntimeError::Protocol(format!( + "{label} {} must not be a symbolic link", + path.display() + ))), + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(io_error("inspect state file")(error)), + } +} + +#[cfg(unix)] +fn verify_owner(metadata: &std::fs::Metadata, path: &Path, label: &str) -> RuntimeResult<()> { + use std::os::unix::fs::MetadataExt; + if metadata.uid() != unsafe { libc::geteuid() } { + return Err(RuntimeError::Protocol(format!( + "Runtime {label} {} is owned by another user", + path.display() + ))); + } + Ok(()) +} + +#[cfg(not(unix))] +fn verify_owner(_metadata: &std::fs::Metadata, _path: &Path, _label: &str) -> RuntimeResult<()> { + Ok(()) +} + +fn verify_owner_only_file( + metadata: &std::fs::Metadata, + path: &Path, + label: &str, +) -> RuntimeResult<()> { + verify_owner(metadata, path, label)?; + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + if metadata.permissions().mode() & 0o077 != 0 || metadata.nlink() != 1 { + return Err(RuntimeError::Protocol(format!( + "Runtime {label} {} is not an owner-only unlinked file", + path.display() + ))); + } + } + Ok(()) +} + +pub(super) fn atomic_write(path: &Path, value: &T, label: &str) -> RuntimeResult<()> { + let parent = path + .parent() + .ok_or_else(|| RuntimeError::Protocol(format!("{label} has no parent")))?; + let bytes = serde_json::to_vec(value) + .map_err(|error| RuntimeError::Protocol(format!("encode {label}: {error}")))?; + + cleanup_stale_staging_files(parent)?; + #[cfg(test)] + super::tests::inject_io_fault("state.atomic-write.before-create") + .map_err(io_error("create state staging file"))?; + + let target_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| RuntimeError::Protocol(format!("{label} has an invalid file name")))?; + let staging_prefix = format!(".a3s-runtime-{target_name}-"); + let mut temporary = tempfile::Builder::new() + .prefix(&staging_prefix) + .suffix(".tmp") + .tempfile_in(parent) + .map_err(io_error("create state staging file"))?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-create"); + + let split = bytes.len().div_ceil(2); + temporary + .write_all(&bytes[..split]) + .map_err(io_error("write state file"))?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-partial-write"); + temporary + .write_all(&bytes[split..]) + .map_err(io_error("write state file"))?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-complete-write"); + temporary + .as_file() + .sync_all() + .map_err(io_error("sync state file"))?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-file-sync"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + temporary + .as_file() + .set_permissions(std::fs::Permissions::from_mode(0o600)) + .map_err(io_error("secure state file"))?; + } + #[cfg(test)] + test_failpoint("state.atomic-write.after-permissions"); + temporary + .as_file() + .sync_all() + .map_err(io_error("sync secured state file"))?; + temporary + .persist(path) + .map_err(|error| io_error("publish state file")(error.error))?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-publish"); + sync_directory(parent)?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-directory-sync"); + Ok(()) +} + +fn cleanup_stale_staging_files(parent: &Path) -> RuntimeResult<()> { + let mut removed = false; + for entry in std::fs::read_dir(parent).map_err(io_error("scan state staging files"))? { + let entry = entry.map_err(io_error("scan state staging file"))?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !name.starts_with(".tmp") + && !(name.starts_with(".a3s-runtime-") && name.ends_with(".tmp")) + { + continue; + } + + let path = entry.path(); + let metadata = + std::fs::symlink_metadata(&path).map_err(io_error("inspect state staging file"))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(RuntimeError::Protocol(format!( + "Runtime state staging file {} is not a regular file", + path.display() + ))); + } + verify_owner_only_file(&metadata, &path, "state staging file")?; + std::fs::remove_file(&path).map_err(io_error("remove stale state staging file"))?; + removed = true; + } + if removed { + sync_directory(parent)?; + } + Ok(()) +} + +#[cfg(unix)] +fn sync_directory(path: &Path) -> RuntimeResult<()> { + File::open(path) + .and_then(|directory| directory.sync_all()) + .map_err(io_error("sync state directory")) +} + +#[cfg(not(unix))] +fn sync_directory(_path: &Path) -> RuntimeResult<()> { + Ok(()) +} + +pub(super) fn io_error(action: &'static str) -> impl FnOnce(std::io::Error) -> RuntimeError { + move |error| RuntimeError::Transport(format!("could not {action}: {error}")) +} + +#[cfg(test)] +fn test_failpoint(name: &str) { + super::tests::hit_failpoint(name); +} diff --git a/src/state/file/tests.rs b/src/state/file/tests.rs index 4f4a8b4..e3ccda4 100644 --- a/src/state/file/tests.rs +++ b/src/state/file/tests.rs @@ -19,6 +19,7 @@ const STOP_ID: &str = "receipt-first-stop"; const REMOVE_ID: &str = "receipt-first-remove"; const EXEC_ID: &str = "receipt-first-exec"; const FAILPOINT_ENV: &str = "A3S_RUNTIME_TEST_FAILPOINT"; +const IO_FAULT_ENV: &str = "A3S_RUNTIME_TEST_IO_FAULT"; const READY_ENV: &str = "A3S_RUNTIME_TEST_FAILPOINT_READY"; const ROOT_ENV: &str = "A3S_RUNTIME_TEST_STATE_ROOT"; const OPERATION_ENV: &str = "A3S_RUNTIME_TEST_COMPLETION_KIND"; @@ -37,6 +38,38 @@ pub(super) fn hit_failpoint(name: &str) { } } +pub(super) fn inject_io_fault(point: &str) -> std::io::Result<()> { + let Ok(fault) = std::env::var(IO_FAULT_ENV) else { + return Ok(()); + }; + let Some(kind) = fault + .strip_prefix(point) + .and_then(|suffix| suffix.strip_prefix('.')) + else { + return Ok(()); + }; + + #[cfg(unix)] + { + let raw_error = match kind { + "enospc" | "inode-exhaustion" => libc::ENOSPC, + "read-only" => libc::EROFS, + _ => return Ok(()), + }; + Err(std::io::Error::from_raw_os_error(raw_error)) + } + + #[cfg(not(unix))] + { + match kind { + "enospc" | "inode-exhaustion" | "read-only" => Err(std::io::Error::other(format!( + "injected Runtime state I/O fault: {kind}" + ))), + _ => Ok(()), + } + } +} + fn digest(character: char) -> String { format!("sha256:{}", character.to_string().repeat(64)) } @@ -474,3 +507,210 @@ fn state_fs_002_corrupt_permission_and_non_regular_boundaries_fail_closed() { Err(RuntimeError::Protocol(_)) )); } + +fn spawn_atomic_crash_helper(root: &Path, ready: &Path, failpoint: &str) -> Child { + Command::new(std::env::current_exe().expect("current test executable")) + .arg("subprocess_atomic_write_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env(ROOT_ENV, root) + .env(READY_ENV, ready) + .env(FAILPOINT_ENV, failpoint) + .spawn() + .expect("spawn atomic-write helper") +} + +fn spawn_io_fault_helper(root: &Path, result: &Path, request_id: &str, fault: &str) -> Child { + Command::new(std::env::current_exe().expect("current test executable")) + .arg("subprocess_atomic_io_fault_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env(ROOT_ENV, root) + .env(READY_ENV, result) + .env(OPERATION_ENV, request_id) + .env(IO_FAULT_ENV, fault) + .spawn() + .expect("spawn atomic I/O-fault helper") +} + +fn wait_for_success(child: &mut Child, case_id: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if let Some(status) = child.try_wait().expect("inspect state helper") { + assert!(status.success(), "{case_id}: helper failed: {status}"); + return; + } + if Instant::now() >= deadline { + child.kill().expect("kill timed-out state helper"); + child.wait().expect("wait for timed-out state helper"); + panic!("{case_id}: helper timed out"); + } + thread::sleep(Duration::from_millis(10)); + } +} + +fn staging_files(root: &Path) -> Vec { + let mut directories = vec![root.to_path_buf()]; + let mut staging = Vec::new(); + while let Some(directory) = directories.pop() { + for entry in std::fs::read_dir(directory).expect("scan state test directory") { + let entry = entry.expect("read state test entry"); + let path = entry.path(); + let file_type = entry.file_type().expect("inspect state test entry"); + if file_type.is_dir() { + directories.push(path); + continue; + } + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with(".tmp") + || (name.starts_with(".a3s-runtime-") && name.ends_with(".tmp")) + { + staging.push(path); + } + } + } + staging.sort(); + staging +} + +#[test] +fn subprocess_atomic_write_helper() { + let Ok(root) = std::env::var(ROOT_ENV) else { + return; + }; + let store = FileRuntimeStateStore::new(PathBuf::from(root)); + let mut record = store.load_sync(UNIT_ID).expect("load atomic-write unit"); + record.observation.observed_at_ms = NOW + 2; + record + .validate() + .expect("validate atomic-write replacement"); + let path = store.record_path(UNIT_ID, false).expect("record path"); + atomic_write(&path, &record, "state record").expect("rewrite state record"); +} + +#[test] +fn subprocess_atomic_io_fault_helper() { + let Ok(root) = std::env::var(ROOT_ENV) else { + return; + }; + let request_id = std::env::var(OPERATION_ENV).expect("fault request ID"); + let result = PathBuf::from(std::env::var(READY_ENV).expect("fault result path")); + let store = FileRuntimeStateStore::new(PathBuf::from(root)); + match store.reserve_action_sync(RuntimeActionKind::Stop, action(&request_id), NOW + 2) { + Err(RuntimeError::Transport(error)) => { + std::fs::write(result, error).expect("write injected I/O result"); + } + other => panic!("injected state I/O fault returned {other:?}"), + } +} + +#[cfg(unix)] +#[test] +fn state_crash_002_atomic_write_process_kills_preserve_a_complete_record() { + let cases = [ + ("state.atomic-write.after-create", false), + ("state.atomic-write.after-partial-write", false), + ("state.atomic-write.after-complete-write", false), + ("state.atomic-write.after-file-sync", false), + ("state.atomic-write.after-permissions", false), + ("state.atomic-write.after-publish", true), + ("state.atomic-write.after-directory-sync", true), + ]; + + for (failpoint, published) in cases { + let directory = tempfile::tempdir().expect("atomic-write state root"); + let store = FileRuntimeStateStore::new(directory.path()); + prepare_running(&store); + let ready = directory.path().join(format!("{failpoint}.ready")); + let mut child = spawn_atomic_crash_helper(directory.path(), &ready, failpoint); + kill_at_failpoint(&mut child, &ready, failpoint); + + let record = store.load_sync(UNIT_ID).expect("load post-crash record"); + assert_eq!( + record.observation.observed_at_ms, + if published { NOW + 2 } else { NOW + 1 }, + "{failpoint}: unexpected published state" + ); + let staged = staging_files(directory.path()); + if published { + assert!( + staged.is_empty(), + "{failpoint}: published state left a staging file: {staged:?}" + ); + } else { + assert!( + !staged.is_empty(), + "{failpoint}: pre-publish crash did not leave a recoverable staging file" + ); + } + + let mut recovered = record; + recovered.observation.observed_at_ms = NOW + 3; + recovered.validate().expect("validate recovered record"); + let path = store.record_path(UNIT_ID, false).expect("record path"); + atomic_write(&path, &recovered, "state record").expect("recover atomic state write"); + assert_eq!( + store + .load_sync(UNIT_ID) + .expect("load recovered state") + .observation + .observed_at_ms, + NOW + 3 + ); + assert!( + staging_files(directory.path()).is_empty(), + "{failpoint}: replay left a stale staging file" + ); + } +} + +#[cfg(unix)] +#[test] +fn state_fs_003_storage_faults_preserve_prior_state_and_reject_dispatch() { + let cases = [ + ( + "STATE-FS-003-read-only", + "state.atomic-write.before-create.read-only", + ), + ( + "STATE-FS-003-enospc", + "state.atomic-write.before-create.enospc", + ), + ( + "STATE-FS-003-inode-exhaustion", + "state.atomic-write.before-create.inode-exhaustion", + ), + ]; + + for (index, (case_id, fault)) in cases.into_iter().enumerate() { + let directory = tempfile::tempdir().expect("storage-fault state root"); + let store = FileRuntimeStateStore::new(directory.path()); + let running = prepare_running(&store); + let request_id = format!("storage-fault-{index}"); + let result = directory.path().join(format!("{request_id}.result")); + let mut child = spawn_io_fault_helper(directory.path(), &result, &request_id, fault); + wait_for_success(&mut child, case_id); + + assert!( + result.is_file(), + "{case_id}: helper did not report the fault" + ); + assert_eq!( + store + .load_sync(UNIT_ID) + .expect("load prior record") + .observation, + running, + "{case_id}: failed reservation changed the durable record" + ); + assert!(matches!( + store.load_request_sync(UNIT_ID, &request_id), + Err(RuntimeError::RequestNotFound { .. }) + )); + assert!( + staging_files(directory.path()).is_empty(), + "{case_id}: failed reservation left a staging file" + ); + } +} From 0414c78364d74b88acfafdd7284ec1dd16eaee15 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 14:46:07 +0800 Subject: [PATCH 23/32] fix: expose state failpoint after module split --- src/state/file.rs | 4 +++- src/state/file/fs.rs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/state/file.rs b/src/state/file.rs index b96dd26..76762cc 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -1,3 +1,5 @@ +#[cfg(test)] +use self::fs::test_failpoint; use self::fs::{ atomic_write, ensure_directory, io_error, owner_only_open, path_exists, read_optional_receipt, read_optional_record, read_required_receipt, read_required_record, @@ -15,7 +17,7 @@ use async_trait::async_trait; use fs2::FileExt; use sha2::{Digest, Sha256}; use std::fs::File; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; mod fs; diff --git a/src/state/file/fs.rs b/src/state/file/fs.rs index f197f05..e87f186 100644 --- a/src/state/file/fs.rs +++ b/src/state/file/fs.rs @@ -312,6 +312,6 @@ pub(super) fn io_error(action: &'static str) -> impl FnOnce(std::io::Error) -> R } #[cfg(test)] -fn test_failpoint(name: &str) { +pub(super) fn test_failpoint(name: &str) { super::tests::hit_failpoint(name); } From 113d7f8d7cdb8ca788ec2b6fc278fb12ac73c465 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 14:47:45 +0800 Subject: [PATCH 24/32] style: simplify staging file predicate --- src/state/file/fs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/state/file/fs.rs b/src/state/file/fs.rs index e87f186..a4a54d7 100644 --- a/src/state/file/fs.rs +++ b/src/state/file/fs.rs @@ -270,8 +270,8 @@ fn cleanup_stale_staging_files(parent: &Path) -> RuntimeResult<()> { let entry = entry.map_err(io_error("scan state staging file"))?; let name = entry.file_name(); let name = name.to_string_lossy(); - if !name.starts_with(".tmp") - && !(name.starts_with(".a3s-runtime-") && name.ends_with(".tmp")) + if !(name.starts_with(".tmp") + || name.starts_with(".a3s-runtime-") && name.ends_with(".tmp")) { continue; } From 0afc1f09ce48d108cd73eaf2452efc8784fe5f3b Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 14:59:11 +0800 Subject: [PATCH 25/32] test: cover cross-process generation races --- tests/process_races.rs | 513 ++++++++++++++++++++++++++++++++++ tests/process_races/driver.rs | 462 ++++++++++++++++++++++++++++++ 2 files changed, 975 insertions(+) create mode 100644 tests/process_races.rs create mode 100644 tests/process_races/driver.rs diff --git a/tests/process_races.rs b/tests/process_races.rs new file mode 100644 index 0000000..1eb9b3c --- /dev/null +++ b/tests/process_races.rs @@ -0,0 +1,513 @@ +#[path = "process_races/driver.rs"] +mod driver; + +use a3s_runtime::contract::{ + ArtifactRef, IsolationLevel, NetworkMode, ResourceLimits, RestartPolicy, RuntimeActionRequest, + RuntimeApplyRequest, RuntimeNetworkSpec, RuntimeProcessSpec, RuntimeUnitClass, RuntimeUnitSpec, + RuntimeUnitState, +}; +use a3s_runtime::{ + FileRuntimeStateStore, ManagedRuntimeClient, RuntimeClient, RuntimeError, RuntimeRequestState, + RuntimeStateStore, +}; +use driver::{ProcessRaceDriver, ProviderResource, IMAGE_MEDIA_TYPE}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant}; + +const STATE_ROOT_ENV: &str = "A3S_RUNTIME_PROCESS_STATE_ROOT"; +const PROVIDER_ROOT_ENV: &str = "A3S_RUNTIME_PROCESS_PROVIDER_ROOT"; +const OPERATION_ENV: &str = "A3S_RUNTIME_PROCESS_OPERATION"; +const REQUEST_ID_ENV: &str = "A3S_RUNTIME_PROCESS_REQUEST_ID"; +const UNIT_ID_ENV: &str = "A3S_RUNTIME_PROCESS_UNIT_ID"; +const GENERATION_ENV: &str = "A3S_RUNTIME_PROCESS_GENERATION"; +const START_GATE_ENV: &str = "A3S_RUNTIME_PROCESS_START_GATE"; +const READY_ENV: &str = "A3S_RUNTIME_PROCESS_READY"; +const RESULT_ENV: &str = "A3S_RUNTIME_PROCESS_RESULT"; +const DRIVER_FAILPOINT_ENV: &str = "A3S_RUNTIME_PROCESS_DRIVER_FAILPOINT"; +const DRIVER_FAILPOINT_READY_ENV: &str = "A3S_RUNTIME_PROCESS_DRIVER_FAILPOINT_READY"; +const APPLY_AFTER_PUBLISH: &str = "provider.apply.after-current-publish"; + +fn service_spec(unit_id: &str, generation: u64) -> RuntimeUnitSpec { + RuntimeUnitSpec { + schema: RuntimeUnitSpec::SCHEMA.into(), + unit_id: unit_id.into(), + generation, + class: RuntimeUnitClass::Service, + artifact: ArtifactRef { + uri: format!( + "oci://registry.example/a3s/process-race@sha256:{}", + "a".repeat(64) + ), + digest: format!("sha256:{}", "a".repeat(64)), + media_type: IMAGE_MEDIA_TYPE.into(), + }, + process: RuntimeProcessSpec { + command: vec!["/bin/service".into()], + args: Vec::new(), + working_directory: None, + environment: BTreeMap::new(), + }, + mounts: Vec::new(), + secrets: Vec::new(), + network: RuntimeNetworkSpec { + mode: NetworkMode::None, + ports: Vec::new(), + }, + resources: ResourceLimits { + cpu_millis: 100, + memory_bytes: 64 * 1024 * 1024, + pids: 32, + ephemeral_storage_bytes: None, + execution_timeout_ms: None, + }, + isolation: IsolationLevel::Container, + health: None, + restart: RestartPolicy::Always, + outputs: Vec::new(), + semantics_profile_digest: None, + } +} + +fn apply_request(request_id: &str, unit_id: &str, generation: u64) -> RuntimeApplyRequest { + RuntimeApplyRequest { + schema: RuntimeApplyRequest::SCHEMA.into(), + request_id: request_id.into(), + deadline_at_ms: None, + spec: service_spec(unit_id, generation), + } +} + +fn action_request(request_id: &str, unit_id: &str, generation: u64) -> RuntimeActionRequest { + RuntimeActionRequest { + schema: RuntimeActionRequest::SCHEMA.into(), + request_id: request_id.into(), + unit_id: unit_id.into(), + generation, + deadline_at_ms: None, + } +} + +fn runtime_client(state_root: &Path, provider_root: &Path) -> ManagedRuntimeClient { + ManagedRuntimeClient::new( + Arc::new(FileRuntimeStateStore::new(state_root)), + Arc::new(ProcessRaceDriver::new(provider_root)), + ) +} + +fn test_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .expect("build process-race test runtime") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn subprocess_process_race_operation_helper() { + let Ok(state_root) = std::env::var(STATE_ROOT_ENV) else { + return; + }; + let provider_root = PathBuf::from(std::env::var(PROVIDER_ROOT_ENV).expect("provider root")); + let operation = std::env::var(OPERATION_ENV).expect("operation"); + let request_id = std::env::var(REQUEST_ID_ENV).expect("request ID"); + let unit_id = std::env::var(UNIT_ID_ENV).expect("unit ID"); + let generation = std::env::var(GENERATION_ENV) + .expect("generation") + .parse::() + .expect("numeric generation"); + let result = PathBuf::from(std::env::var(RESULT_ENV).expect("result path")); + + if let Ok(ready) = std::env::var(READY_ENV) { + std::fs::write(ready, b"ready").expect("publish process helper readiness"); + } + if let Ok(gate) = std::env::var(START_GATE_ENV) { + let gate = PathBuf::from(gate); + let deadline = Instant::now() + Duration::from_secs(10); + while !gate.is_file() { + assert!( + Instant::now() < deadline, + "process helper start gate timed out" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + let client = runtime_client(Path::new(&state_root), &provider_root); + let outcome = match operation.as_str() { + "apply" => client + .apply(&apply_request(&request_id, &unit_id, generation)) + .await + .map(|_| ()), + "stop" => client + .stop(&action_request(&request_id, &unit_id, generation)) + .await + .map(|_| ()), + "remove" => client + .remove(&action_request(&request_id, &unit_id, generation)) + .await + .map(|_| ()), + other => panic!("unknown process helper operation {other:?}"), + }; + std::fs::write(result, classify(outcome)).expect("write process helper result"); +} + +fn classify(result: Result<(), RuntimeError>) -> &'static str { + match result { + Ok(()) => "ok", + Err(RuntimeError::StaleGeneration { .. }) => "stale-generation", + Err(RuntimeError::GenerationConflict { .. }) => "generation-conflict", + Err(RuntimeError::NotFound { .. }) => "not-found", + Err(RuntimeError::Protocol(_)) => "protocol", + Err(_) => "unexpected-error", + } +} + +struct ProcessCase { + child: Child, + ready: PathBuf, + result: PathBuf, +} + +fn spawn_operation( + state_root: &Path, + provider_root: &Path, + gate: &Path, + operation: &str, + request_id: &str, + unit_id: &str, + generation: u64, +) -> ProcessCase { + let result = state_root.join(format!("{request_id}.result")); + let ready = state_root.join(format!("{request_id}.ready")); + let child = Command::new(std::env::current_exe().expect("current test executable")) + .arg("subprocess_process_race_operation_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env(STATE_ROOT_ENV, state_root) + .env(PROVIDER_ROOT_ENV, provider_root) + .env(OPERATION_ENV, operation) + .env(REQUEST_ID_ENV, request_id) + .env(UNIT_ID_ENV, unit_id) + .env(GENERATION_ENV, generation.to_string()) + .env(START_GATE_ENV, gate) + .env(READY_ENV, &ready) + .env(RESULT_ENV, &result) + .spawn() + .expect("spawn process race operation"); + ProcessCase { + child, + ready, + result, + } +} + +fn wait_for_ready(case: &mut ProcessCase, case_id: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + while !case.ready.is_file() { + if let Some(status) = case.child.try_wait().expect("inspect process helper") { + panic!("{case_id}: helper exited before ready: {status}"); + } + assert!(Instant::now() < deadline, "{case_id}: readiness timed out"); + thread::sleep(Duration::from_millis(10)); + } +} + +fn wait_for_exit(case: &mut ProcessCase, case_id: &str) -> String { + let deadline = Instant::now() + Duration::from_secs(20); + loop { + if let Some(status) = case.child.try_wait().expect("inspect process helper") { + assert!(status.success(), "{case_id}: helper failed: {status}"); + return std::fs::read_to_string(&case.result).expect("read process helper result"); + } + if Instant::now() >= deadline { + case.child.kill().expect("kill timed-out process helper"); + case.child + .wait() + .expect("wait for timed-out process helper"); + panic!("{case_id}: helper timed out"); + } + thread::sleep(Duration::from_millis(10)); + } +} + +fn run_pair( + state_root: &Path, + provider_root: &Path, + case_id: &str, + left: (&str, &str, &str, u64), + right: (&str, &str, &str, u64), +) -> (String, String) { + let gate = state_root.join(format!("{case_id}.gate")); + let mut left_case = spawn_operation( + state_root, + provider_root, + &gate, + left.0, + left.1, + left.2, + left.3, + ); + let mut right_case = spawn_operation( + state_root, + provider_root, + &gate, + right.0, + right.1, + right.2, + right.3, + ); + wait_for_ready(&mut left_case, case_id); + wait_for_ready(&mut right_case, case_id); + std::fs::write(&gate, b"start").expect("release process race gate"); + ( + wait_for_exit(&mut left_case, case_id), + wait_for_exit(&mut right_case, case_id), + ) +} + +fn assert_single_generation(resources: &[ProviderResource], generation: u64) { + assert_eq!(resources.len(), 1, "provider inventory is not singular"); + assert_eq!(resources[0].generation, generation); +} + +#[test] +fn race_gen_001_concurrent_generations_converge_to_one_newest_resource() { + let state = tempfile::tempdir().expect("generation-race state root"); + let provider = tempfile::tempdir().expect("generation-race provider root"); + let unit_id = "race-generation-unit"; + let (generation_one, generation_two) = run_pair( + state.path(), + provider.path(), + "RACE-GEN-001", + ("apply", "race-generation-one", unit_id, 1), + ("apply", "race-generation-two", unit_id, 2), + ); + assert!(matches!(generation_one.as_str(), "ok" | "stale-generation")); + assert_eq!(generation_two, "ok"); + + let runtime = test_runtime(); + let store = FileRuntimeStateStore::new(state.path()); + let record = runtime + .block_on(store.load(unit_id)) + .expect("load final unit"); + assert_eq!(record.spec.generation, 2); + assert_eq!(record.observation.state, RuntimeUnitState::Running); + assert_single_generation( + &ProcessRaceDriver::new(provider.path()) + .inventory(unit_id) + .expect("final provider inventory"), + 2, + ); +} + +#[test] +fn race_ops_001_apply_stop_remove_process_races_have_deterministic_oracles() { + for (suffix, competing_operation) in [("stop", "stop"), ("remove", "remove")] { + let state = tempfile::tempdir().expect("operation-race state root"); + let provider = tempfile::tempdir().expect("operation-race provider root"); + let unit_id = format!("race-apply-{suffix}"); + let runtime = test_runtime(); + runtime + .block_on( + runtime_client(state.path(), provider.path()).apply(&apply_request( + &format!("prepare-{suffix}"), + &unit_id, + 1, + )), + ) + .expect("prepare operation race"); + let (apply_result, action_result) = run_pair( + state.path(), + provider.path(), + &format!("RACE-OPS-001-apply-{suffix}"), + ("apply", &format!("apply-{suffix}-g2"), &unit_id, 2), + (competing_operation, &format!("{suffix}-g1"), &unit_id, 1), + ); + assert_eq!(apply_result, "ok"); + assert!(matches!( + action_result.as_str(), + "ok" | "stale-generation" | "not-found" + )); + let record = runtime + .block_on(FileRuntimeStateStore::new(state.path()).load(&unit_id)) + .expect("load apply/action race unit"); + assert_eq!(record.spec.generation, 2); + assert_eq!(record.observation.state, RuntimeUnitState::Running); + assert_single_generation( + &ProcessRaceDriver::new(provider.path()) + .inventory(&unit_id) + .expect("apply/action provider inventory"), + 2, + ); + } + + let state = tempfile::tempdir().expect("stop-remove state root"); + let provider = tempfile::tempdir().expect("stop-remove provider root"); + let unit_id = "race-stop-remove"; + let runtime = test_runtime(); + runtime + .block_on( + runtime_client(state.path(), provider.path()).apply(&apply_request( + "prepare-stop-remove", + unit_id, + 1, + )), + ) + .expect("prepare stop/remove race"); + let (stop_result, remove_result) = run_pair( + state.path(), + provider.path(), + "RACE-OPS-001-stop-remove", + ("stop", "race-stop", unit_id, 1), + ("remove", "race-remove", unit_id, 1), + ); + assert!(matches!(stop_result.as_str(), "ok" | "not-found")); + assert_eq!(remove_result, "ok"); + let record = runtime + .block_on(FileRuntimeStateStore::new(state.path()).load(unit_id)) + .expect("load stop/remove race unit"); + assert!(record.removed_at_ms.is_some()); + assert!(ProcessRaceDriver::new(provider.path()) + .inventory(unit_id) + .expect("stop/remove provider inventory") + .is_empty()); +} + +#[test] +fn crash_gen_001_killed_generation_handoff_exact_replay_converges_once() { + let state = tempfile::tempdir().expect("generation-crash state root"); + let provider = tempfile::tempdir().expect("generation-crash provider root"); + let unit_id = "generation-crash-unit"; + let runtime = test_runtime(); + runtime + .block_on( + runtime_client(state.path(), provider.path()).apply(&apply_request( + "generation-crash-one", + unit_id, + 1, + )), + ) + .expect("prepare generation crash"); + + let result = state.path().join("generation-crash-two.result"); + let failpoint_ready = state.path().join("generation-crash-two.failpoint"); + let mut child = Command::new(std::env::current_exe().expect("current test executable")) + .arg("subprocess_process_race_operation_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env(STATE_ROOT_ENV, state.path()) + .env(PROVIDER_ROOT_ENV, provider.path()) + .env(OPERATION_ENV, "apply") + .env(REQUEST_ID_ENV, "generation-crash-two") + .env(UNIT_ID_ENV, unit_id) + .env(GENERATION_ENV, "2") + .env(RESULT_ENV, &result) + .env(DRIVER_FAILPOINT_ENV, APPLY_AFTER_PUBLISH) + .env(DRIVER_FAILPOINT_READY_ENV, &failpoint_ready) + .spawn() + .expect("spawn generation crash helper"); + let deadline = Instant::now() + Duration::from_secs(10); + while !failpoint_ready.is_file() { + if let Some(status) = child.try_wait().expect("inspect generation crash helper") { + panic!("generation crash helper exited early: {status}"); + } + assert!(Instant::now() < deadline, "generation failpoint timed out"); + thread::sleep(Duration::from_millis(10)); + } + child.kill().expect("kill generation crash helper"); + assert!(!child + .wait() + .expect("wait generation crash helper") + .success()); + + let store = FileRuntimeStateStore::new(state.path()); + let interrupted = runtime + .block_on(store.load(unit_id)) + .expect("load interrupted unit"); + assert_eq!(interrupted.spec.generation, 2); + assert_eq!(interrupted.observation.state, RuntimeUnitState::Accepted); + assert_eq!( + runtime + .block_on(store.load_request(unit_id, "generation-crash-two")) + .expect("load interrupted receipt") + .state, + RuntimeRequestState::Pending + ); + let process_driver = ProcessRaceDriver::new(provider.path()); + let interrupted_inventory = process_driver + .inventory(unit_id) + .expect("interrupted provider inventory"); + assert_eq!( + interrupted_inventory + .iter() + .map(|resource| resource.generation) + .collect::>(), + vec![1, 2] + ); + + let request = apply_request("generation-crash-two", unit_id, 2); + let recovered = runtime + .block_on(runtime_client(state.path(), provider.path()).apply(&request)) + .expect("recover generation handoff"); + assert_eq!(recovered.state, RuntimeUnitState::Running); + let converged = process_driver + .inventory(unit_id) + .expect("converged provider inventory"); + assert_single_generation(&converged, 2); + assert_eq!(converged[0].apply_dispatches, 2); + let replayed = runtime + .block_on(runtime_client(state.path(), provider.path()).apply(&request)) + .expect("replay recovered generation"); + assert_eq!(replayed, recovered); + assert_eq!( + process_driver + .inventory(unit_id) + .expect("replayed provider inventory"), + converged + ); +} + +#[test] +fn race_inventory_001_duplicate_resources_fail_closed_before_provider_mutation() { + let state = tempfile::tempdir().expect("duplicate state root"); + let provider = tempfile::tempdir().expect("duplicate provider root"); + let unit_id = "duplicate-provider-unit"; + let runtime = test_runtime(); + let client = runtime_client(state.path(), provider.path()); + runtime + .block_on(client.apply(&apply_request("duplicate-one", unit_id, 1))) + .expect("prepare duplicate provider fixture"); + let process_driver = ProcessRaceDriver::new(provider.path()); + process_driver + .inject_duplicate(unit_id, 1) + .expect("inject duplicate provider resource"); + assert_eq!( + process_driver + .inventory(unit_id) + .expect("duplicate provider inventory") + .len(), + 2 + ); + assert!(matches!( + runtime.block_on(client.inspect(unit_id)), + Err(RuntimeError::Protocol(message)) if message.contains("duplicate resources") + )); + assert!(matches!( + runtime.block_on(client.apply(&apply_request("duplicate-two", unit_id, 2))), + Err(RuntimeError::Protocol(message)) if message.contains("duplicate resources") + )); + let resources = process_driver + .inventory(unit_id) + .expect("post-rejection provider inventory"); + assert_eq!(resources.len(), 2); + assert!(resources.iter().all(|resource| resource.generation == 1)); + let record = runtime + .block_on(FileRuntimeStateStore::new(state.path()).load(unit_id)) + .expect("load duplicate-rejection state"); + assert_eq!(record.spec.generation, 2); + assert_eq!(record.observation.state, RuntimeUnitState::Accepted); +} diff --git a/tests/process_races/driver.rs b/tests/process_races/driver.rs new file mode 100644 index 0000000..644a284 --- /dev/null +++ b/tests/process_races/driver.rs @@ -0,0 +1,462 @@ +use a3s_runtime::contract::{ + IsolationLevel, NetworkMode, ResourceControl, RuntimeActionRequest, RuntimeCapabilities, + RuntimeExecRequest, RuntimeExecResult, RuntimeFeature, RuntimeInspection, RuntimeLogChunk, + RuntimeLogQuery, RuntimeObservation, RuntimeRemoval, RuntimeUnitClass, RuntimeUnitSpec, + RuntimeUnitState, +}; +use a3s_runtime::{ProviderId, RuntimeDriver, RuntimeError, RuntimeResult, RuntimeUnitRecord}; +use async_trait::async_trait; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub const IMAGE_MEDIA_TYPE: &str = "application/vnd.oci.image.manifest.v1+json"; +const PROVIDER_BUILD: &str = "process-race-driver/1"; +const FAILPOINT_ENV: &str = "A3S_RUNTIME_PROCESS_DRIVER_FAILPOINT"; +const FAILPOINT_READY_ENV: &str = "A3S_RUNTIME_PROCESS_DRIVER_FAILPOINT_READY"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProviderResource { + pub resource_id: String, + pub unit_id: String, + pub generation: u64, + pub state: RuntimeUnitState, + pub observed_at_ms: u64, + pub started_at_ms: u64, + pub apply_dispatches: u64, +} + +#[derive(Debug, Clone)] +pub struct ProcessRaceDriver { + root: PathBuf, + provider_id: ProviderId, +} + +impl ProcessRaceDriver { + pub fn new(root: impl Into) -> Self { + Self { + root: root.into(), + provider_id: ProviderId::parse("process-race-runtime") + .expect("valid process-race provider ID"), + } + } + + pub fn inventory(&self, unit_id: &str) -> RuntimeResult> { + self.with_lock(|root| inventory_unlocked(root, unit_id)) + } + + pub fn inject_duplicate(&self, unit_id: &str, generation: u64) -> RuntimeResult<()> { + self.with_lock(|root| { + let resources = inventory_unlocked(root, unit_id)?; + let source = resources + .iter() + .find(|resource| resource.generation == generation) + .ok_or_else(|| RuntimeError::Protocol("duplicate source is absent".into()))?; + let mut duplicate = source.clone(); + duplicate.resource_id = format!("{}/duplicate", source.resource_id); + write_resource(root, &duplicate) + }) + } + + fn with_lock(&self, operation: impl FnOnce(&Path) -> RuntimeResult) -> RuntimeResult { + ensure_provider_root(&self.root)?; + let lock_path = self.root.join("provider.lock"); + let mut options = OpenOptions::new(); + options.create(true).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let lock = options + .open(lock_path) + .map_err(io_error("open process provider lock"))?; + lock.lock_exclusive() + .map_err(io_error("lock process provider"))?; + let _guard = ProviderLock(lock); + operation(&self.root) + } + + fn apply_sync( + &self, + spec: &RuntimeUnitSpec, + current: &RuntimeObservation, + ) -> RuntimeResult { + self.with_lock(|root| { + let resources = inventory_unlocked(root, &spec.unit_id)?; + if let Some(generation) = duplicate_generation(&resources) { + return Err(duplicate_error(&spec.unit_id, generation)); + } + let current_resources = resources + .iter() + .filter(|resource| resource.generation == spec.generation) + .collect::>(); + + let mut resource = current_resources.first().map_or_else( + || ProviderResource { + resource_id: format!("process/{}/g{}", spec.unit_id, spec.generation), + unit_id: spec.unit_id.clone(), + generation: spec.generation, + state: RuntimeUnitState::Running, + observed_at_ms: current.observed_at_ms.saturating_add(1), + started_at_ms: current.observed_at_ms.saturating_add(1), + apply_dispatches: 0, + }, + |resource| (*resource).clone(), + ); + resource.apply_dispatches = resource.apply_dispatches.saturating_add(1); + resource.state = RuntimeUnitState::Running; + resource.observed_at_ms = resource + .observed_at_ms + .max(current.observed_at_ms.saturating_add(1)); + write_resource(root, &resource)?; + hit_failpoint("provider.apply.after-current-publish"); + + for stale in resources + .iter() + .filter(|candidate| candidate.generation != spec.generation) + { + remove_resource(root, &stale.resource_id)?; + } + let final_inventory = inventory_unlocked(root, &spec.unit_id)?; + if final_inventory.len() != 1 + || final_inventory[0].generation != spec.generation + || final_inventory[0].resource_id != resource.resource_id + { + return Err(RuntimeError::Protocol(format!( + "process provider did not converge {:?} generation {} to one resource", + spec.unit_id, spec.generation + ))); + } + Ok(observation_from_resource(current, &resource)) + }) + } + + fn inspect_sync(&self, unit: &RuntimeUnitRecord) -> RuntimeResult { + self.with_lock(|root| { + let resources = inventory_unlocked(root, &unit.spec.unit_id)?; + let current = resources + .iter() + .filter(|resource| resource.generation == unit.spec.generation) + .collect::>(); + if current.len() > 1 { + return Err(duplicate_error(&unit.spec.unit_id, unit.spec.generation)); + } + let Some(resource) = current.first() else { + return Ok(RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), + unit_id: unit.spec.unit_id.clone(), + last_generation: Some(unit.spec.generation), + }); + }; + Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), + observation: Box::new(observation_from_resource(&unit.observation, resource)), + }) + }) + } + + fn stop_sync(&self, unit: &RuntimeUnitRecord) -> RuntimeResult { + self.with_lock(|root| { + let resources = inventory_unlocked(root, &unit.spec.unit_id)?; + let mut current = resources + .into_iter() + .filter(|resource| resource.generation == unit.spec.generation) + .collect::>(); + if current.len() > 1 { + return Err(duplicate_error(&unit.spec.unit_id, unit.spec.generation)); + } + let Some(mut resource) = current.pop() else { + let mut unknown = unit.observation.clone(); + unknown.state = RuntimeUnitState::Unknown; + unknown.observed_at_ms = unknown.observed_at_ms.saturating_add(1); + unknown.finished_at_ms = None; + unknown.health = None; + unknown.outputs.clear(); + unknown.failure = None; + return Ok(unknown); + }; + resource.state = RuntimeUnitState::Stopped; + resource.observed_at_ms = resource + .observed_at_ms + .max(unit.observation.observed_at_ms.saturating_add(1)); + write_resource(root, &resource)?; + Ok(observation_from_resource(&unit.observation, &resource)) + }) + } + + fn remove_sync( + &self, + unit: &RuntimeUnitRecord, + request: &RuntimeActionRequest, + ) -> RuntimeResult { + self.with_lock(|root| { + let resources = inventory_unlocked(root, &unit.spec.unit_id)?; + let current = resources + .iter() + .filter(|resource| resource.generation == unit.spec.generation) + .collect::>(); + if current.len() > 1 { + return Err(duplicate_error(&unit.spec.unit_id, unit.spec.generation)); + } + let already_absent = current.is_empty(); + if let Some(resource) = current.first() { + remove_resource(root, &resource.resource_id)?; + } + Ok(RuntimeRemoval { + schema: RuntimeRemoval::SCHEMA.into(), + request_id: request.request_id.clone(), + unit_id: request.unit_id.clone(), + generation: request.generation, + removed_at_ms: now_ms(), + already_absent, + }) + }) + } +} + +#[async_trait] +impl RuntimeDriver for ProcessRaceDriver { + fn provider_id(&self) -> &ProviderId { + &self.provider_id + } + + async fn capabilities(&self) -> RuntimeResult { + Ok(RuntimeCapabilities { + schema: RuntimeCapabilities::SCHEMA.into(), + provider_id: self.provider_id.clone(), + provider_build: PROVIDER_BUILD.into(), + unit_classes: vec![RuntimeUnitClass::Service], + artifact_media_types: vec![IMAGE_MEDIA_TYPE.into()], + isolation_levels: vec![IsolationLevel::Container], + network_modes: vec![NetworkMode::None], + mount_kinds: Vec::new(), + health_check_kinds: Vec::new(), + resource_controls: vec![ + ResourceControl::Cpu, + ResourceControl::Memory, + ResourceControl::Pids, + ], + features: vec![ + RuntimeFeature::DurableIdentity, + RuntimeFeature::Stop, + RuntimeFeature::Remove, + ], + }) + } + + async fn apply( + &self, + spec: &RuntimeUnitSpec, + current: &RuntimeObservation, + ) -> RuntimeResult { + let driver = self.clone(); + let spec = spec.clone(); + let current = current.clone(); + tokio::task::spawn_blocking(move || driver.apply_sync(&spec, ¤t)) + .await + .map_err(task_error)? + } + + async fn inspect(&self, unit: &RuntimeUnitRecord) -> RuntimeResult { + let driver = self.clone(); + let unit = unit.clone(); + tokio::task::spawn_blocking(move || driver.inspect_sync(&unit)) + .await + .map_err(task_error)? + } + + async fn stop( + &self, + unit: &RuntimeUnitRecord, + _request: &RuntimeActionRequest, + ) -> RuntimeResult { + let driver = self.clone(); + let unit = unit.clone(); + tokio::task::spawn_blocking(move || driver.stop_sync(&unit)) + .await + .map_err(task_error)? + } + + async fn remove( + &self, + unit: &RuntimeUnitRecord, + request: &RuntimeActionRequest, + ) -> RuntimeResult { + let driver = self.clone(); + let unit = unit.clone(); + let request = request.clone(); + tokio::task::spawn_blocking(move || driver.remove_sync(&unit, &request)) + .await + .map_err(task_error)? + } + + async fn logs( + &self, + _unit: &RuntimeUnitRecord, + _query: &RuntimeLogQuery, + ) -> RuntimeResult> { + Err(RuntimeError::UnsupportedCapabilities(vec![ + "feature:Logs".into() + ])) + } + + async fn exec( + &self, + _unit: &RuntimeUnitRecord, + _request: &RuntimeExecRequest, + ) -> RuntimeResult { + Err(RuntimeError::UnsupportedCapabilities(vec![ + "feature:Exec".into() + ])) + } +} + +fn ensure_provider_root(root: &Path) -> RuntimeResult<()> { + std::fs::create_dir_all(root.join("resources")) + .map_err(io_error("create process provider root"))?; + Ok(()) +} + +fn inventory_unlocked(root: &Path, unit_id: &str) -> RuntimeResult> { + let mut resources = Vec::new(); + for entry in std::fs::read_dir(root.join("resources")) + .map_err(io_error("read process provider inventory"))? + { + let entry = entry.map_err(io_error("read process provider resource"))?; + if entry.path().extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let bytes = std::fs::read(entry.path()).map_err(io_error("read provider resource"))?; + let resource: ProviderResource = serde_json::from_slice(&bytes).map_err(|error| { + RuntimeError::Protocol(format!("invalid provider resource: {error}")) + })?; + if resource.unit_id == unit_id { + resources.push(resource); + } + } + resources.sort_by(|left, right| { + (left.generation, &left.resource_id).cmp(&(right.generation, &right.resource_id)) + }); + Ok(resources) +} + +fn write_resource(root: &Path, resource: &ProviderResource) -> RuntimeResult<()> { + let path = resource_path(root, &resource.resource_id); + let temporary = path.with_extension(format!("{}.tmp", std::process::id())); + let bytes = serde_json::to_vec(resource) + .map_err(|error| RuntimeError::Protocol(format!("encode provider resource: {error}")))?; + let mut options = OpenOptions::new(); + options.create(true).truncate(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(&temporary) + .map_err(io_error("create provider staging file"))?; + file.write_all(&bytes) + .and_then(|()| file.sync_all()) + .map_err(io_error("write provider resource"))?; + std::fs::rename(&temporary, &path).map_err(io_error("publish provider resource"))?; + sync_directory(&root.join("resources")) +} + +fn remove_resource(root: &Path, resource_id: &str) -> RuntimeResult<()> { + let path = resource_path(root, resource_id); + match std::fs::remove_file(path) { + Ok(()) => sync_directory(&root.join("resources")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(io_error("remove provider resource")(error)), + } +} + +fn resource_path(root: &Path, resource_id: &str) -> PathBuf { + let key = format!("{:x}", Sha256::digest(resource_id.as_bytes())); + root.join("resources").join(format!("{key}.json")) +} + +fn observation_from_resource( + current: &RuntimeObservation, + resource: &ProviderResource, +) -> RuntimeObservation { + let mut observation = current.clone(); + observation.state = resource.state; + observation.provider_resource_id = Some(resource.resource_id.clone()); + observation.provider_build = Some(PROVIDER_BUILD.into()); + observation.observed_at_ms = resource.observed_at_ms.max(current.observed_at_ms); + observation.started_at_ms = Some(resource.started_at_ms); + observation.finished_at_ms = resource + .state + .is_terminal() + .then_some(observation.observed_at_ms); + observation.health = None; + observation.outputs.clear(); + observation.failure = None; + observation +} + +fn duplicate_error(unit_id: &str, generation: u64) -> RuntimeError { + RuntimeError::Protocol(format!( + "process provider found duplicate resources for {unit_id:?} generation {generation}" + )) +} + +fn duplicate_generation(resources: &[ProviderResource]) -> Option { + resources + .windows(2) + .find_map(|pair| (pair[0].generation == pair[1].generation).then_some(pair[0].generation)) +} + +fn hit_failpoint(name: &str) { + if !matches!(std::env::var(FAILPOINT_ENV), Ok(value) if value == name) { + return; + } + let ready = std::env::var(FAILPOINT_READY_ENV).expect("provider failpoint ready path"); + std::fs::write(ready, name).expect("publish provider failpoint readiness"); + loop { + std::thread::park_timeout(std::time::Duration::from_secs(60)); + } +} + +fn now_ms() -> u64 { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + u64::try_from(millis).unwrap_or(u64::MAX) +} + +#[cfg(unix)] +fn sync_directory(path: &Path) -> RuntimeResult<()> { + File::open(path) + .and_then(|directory| directory.sync_all()) + .map_err(io_error("sync process provider directory")) +} + +#[cfg(not(unix))] +fn sync_directory(_path: &Path) -> RuntimeResult<()> { + Ok(()) +} + +fn task_error(error: tokio::task::JoinError) -> RuntimeError { + RuntimeError::Transport(format!("process provider task failed: {error}")) +} + +fn io_error(action: &'static str) -> impl FnOnce(std::io::Error) -> RuntimeError { + move |error| RuntimeError::Transport(format!("could not {action}: {error}")) +} + +struct ProviderLock(File); + +impl Drop for ProviderLock { + fn drop(&mut self) { + let _ = FileExt::unlock(&self.0); + } +} From ce07b1668912b6b1c40e6ff671df561acc55617f Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 15:11:37 +0800 Subject: [PATCH 26/32] feat: add capability-driven conformance profiles --- src/conformance.rs | 249 +++++++++++++++++++++ src/conformance/profiles.rs | 395 ++++++++++++++++++++++++++++++++++ src/lib.rs | 8 +- tests/conformance_profiles.rs | 318 +++++++++++++++++++++++++++ tests/process_races/driver.rs | 43 ++-- 5 files changed, 1000 insertions(+), 13 deletions(-) create mode 100644 src/conformance/profiles.rs create mode 100644 tests/conformance_profiles.rs diff --git a/src/conformance.rs b/src/conformance.rs index b8fc2e7..0d40c37 100644 --- a/src/conformance.rs +++ b/src/conformance.rs @@ -3,6 +3,16 @@ use crate::contract::{ RuntimeRemoval, RuntimeUnitClass, RuntimeUnitState, }; use crate::{RuntimeClient, RuntimeError, RuntimeResult}; +use std::collections::BTreeSet; + +mod profiles; + +pub use profiles::{ + required_runtime_profiles, runtime_profile_requirements, verify_runtime_profiles, + RuntimeConformanceFixture, RuntimeConformanceInventory, RuntimeConformanceProfile, + RuntimeConformanceProfileEvidence, RuntimeConformanceProfileRequirements, + RuntimeConformanceSuiteReport, +}; /// Provider-owned inputs for the destructive Runtime conformance suite. /// @@ -62,6 +72,122 @@ pub struct RuntimeConformanceReport { pub service_removal: RuntimeRemoval, } +/// Complete provider-owned inputs for the mandatory Base profile. +/// +/// Failure and timeout Tasks must deterministically return a `failed` +/// observation. The generation-conflict pair must target the same unit and +/// generation with different canonical specifications. +#[derive(Debug, Clone)] +pub struct RuntimeBaseConformanceCase { + pub lifecycle: RuntimeConformanceCase, + pub task_failure_apply: RuntimeApplyRequest, + pub task_failure_remove: RuntimeActionRequest, + pub task_timeout_apply: RuntimeApplyRequest, + pub task_timeout_remove: RuntimeActionRequest, + pub generation_apply: RuntimeApplyRequest, + pub generation_conflict_apply: RuntimeApplyRequest, + pub generation_remove: RuntimeActionRequest, +} + +impl RuntimeBaseConformanceCase { + pub fn validate(&self) -> Result<(), String> { + self.lifecycle.validate()?; + for (label, apply, remove) in [ + ( + "failure", + &self.task_failure_apply, + &self.task_failure_remove, + ), + ( + "timeout", + &self.task_timeout_apply, + &self.task_timeout_remove, + ), + ] { + apply.validate()?; + remove.validate()?; + if apply.spec.class != RuntimeUnitClass::Task { + return Err(format!("conformance {label} fixture must describe a Task")); + } + validate_action(remove, apply)?; + } + + self.generation_apply.validate()?; + self.generation_conflict_apply.validate()?; + self.generation_remove.validate()?; + if self.generation_apply.spec.class != RuntimeUnitClass::Service { + return Err("conformance generation fixture must describe a Service".into()); + } + if self.generation_apply.spec.unit_id != self.generation_conflict_apply.spec.unit_id + || self.generation_apply.spec.generation + != self.generation_conflict_apply.spec.generation + { + return Err( + "conformance generation-conflict fixtures must target one generation".into(), + ); + } + if self.generation_apply.spec.digest()? == self.generation_conflict_apply.spec.digest()? { + return Err( + "conformance generation-conflict fixtures must have different content".into(), + ); + } + validate_action(&self.generation_remove, &self.generation_apply)?; + + let unit_ids = [ + self.lifecycle.task_apply.spec.unit_id.as_str(), + self.lifecycle.service_apply.spec.unit_id.as_str(), + self.task_failure_apply.spec.unit_id.as_str(), + self.task_timeout_apply.spec.unit_id.as_str(), + self.generation_apply.spec.unit_id.as_str(), + ]; + if unit_ids.iter().copied().collect::>().len() != unit_ids.len() { + return Err("Base conformance fixtures must use distinct unit IDs".into()); + } + + let request_ids = [ + self.lifecycle.task_apply.request_id.as_str(), + self.lifecycle.task_remove.request_id.as_str(), + self.lifecycle.service_apply.request_id.as_str(), + self.lifecycle.service_stop.request_id.as_str(), + self.lifecycle.service_remove.request_id.as_str(), + self.task_failure_apply.request_id.as_str(), + self.task_failure_remove.request_id.as_str(), + self.task_timeout_apply.request_id.as_str(), + self.task_timeout_remove.request_id.as_str(), + self.generation_apply.request_id.as_str(), + self.generation_conflict_apply.request_id.as_str(), + self.generation_remove.request_id.as_str(), + ]; + if request_ids.iter().copied().collect::>().len() != request_ids.len() { + return Err("Base conformance fixtures must use unique request IDs".into()); + } + Ok(()) + } + + pub(crate) fn specifications(&self) -> [&crate::contract::RuntimeUnitSpec; 6] { + [ + &self.lifecycle.task_apply.spec, + &self.lifecycle.service_apply.spec, + &self.task_failure_apply.spec, + &self.task_timeout_apply.spec, + &self.generation_apply.spec, + &self.generation_conflict_apply.spec, + ] + } +} + +/// Evidence returned after the complete mandatory Base profile passes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeBaseConformanceReport { + pub lifecycle: RuntimeConformanceReport, + pub failed_task: RuntimeObservation, + pub failed_task_removal: RuntimeRemoval, + pub timed_out_task: RuntimeObservation, + pub timed_out_task_removal: RuntimeRemoval, + pub generation: RuntimeObservation, + pub generation_removal: RuntimeRemoval, +} + /// Runs the destructive provider-neutral lifecycle conformance suite. /// /// Provider-specific tests remain responsible for crash injection and resource @@ -161,6 +287,129 @@ pub async fn verify_runtime_provider( }) } +/// Runs every mandatory Base-profile oracle, including negative Task results +/// and a same-generation content conflict. +pub async fn verify_runtime_base( + client: &dyn RuntimeClient, + case: &RuntimeBaseConformanceCase, +) -> RuntimeResult { + case.validate().map_err(RuntimeError::InvalidRequest)?; + let capabilities = client.capabilities().await?; + capabilities.validate().map_err(RuntimeError::Protocol)?; + for spec in case.specifications() { + let missing = capabilities + .missing_for(spec) + .map_err(RuntimeError::Protocol)?; + if !missing.is_empty() { + return Err(RuntimeError::UnsupportedCapabilities(missing)); + } + } + + let lifecycle = verify_runtime_provider(client, &case.lifecycle).await?; + let (failed_task, failed_task_removal) = verify_failed_task( + client, + "failed Task", + &case.task_failure_apply, + &case.task_failure_remove, + ) + .await?; + let (timed_out_task, timed_out_task_removal) = verify_failed_task( + client, + "timed-out Task", + &case.task_timeout_apply, + &case.task_timeout_remove, + ) + .await?; + + let generation = client.apply(&case.generation_apply).await?; + generation + .validate_against(&case.generation_apply.spec) + .map_err(RuntimeError::Protocol)?; + if !generation.converges(&case.generation_apply.spec) { + return Err(RuntimeError::Protocol( + "Base generation fixture did not converge".into(), + )); + } + require_equal( + "duplicate generation apply", + &generation, + &client.apply(&case.generation_apply).await?, + )?; + match client.apply(&case.generation_conflict_apply).await { + Err(RuntimeError::GenerationConflict { + unit_id, + generation: rejected_generation, + }) if unit_id == case.generation_apply.spec.unit_id + && rejected_generation == case.generation_apply.spec.generation => {} + Err(error) => return Err(error), + Ok(_) => { + return Err(RuntimeError::Protocol( + "Base generation conflict unexpectedly succeeded".into(), + )); + } + } + let generation_removal = client.remove(&case.generation_remove).await?; + require_equal( + "duplicate generation removal", + &generation_removal, + &client.remove(&case.generation_remove).await?, + )?; + require_absent( + "removed generation fixture inspection", + client.inspect(&generation.unit_id).await?, + generation.generation, + )?; + + Ok(RuntimeBaseConformanceReport { + lifecycle, + failed_task, + failed_task_removal, + timed_out_task, + timed_out_task_removal, + generation, + generation_removal, + }) +} + +async fn verify_failed_task( + client: &dyn RuntimeClient, + label: &str, + apply: &RuntimeApplyRequest, + remove: &RuntimeActionRequest, +) -> RuntimeResult<(RuntimeObservation, RuntimeRemoval)> { + let observation = client.apply(apply).await?; + observation + .validate_against(&apply.spec) + .map_err(RuntimeError::Protocol)?; + if observation.state != RuntimeUnitState::Failed { + return Err(RuntimeError::Protocol(format!( + "conformance {label} did not reach failed" + ))); + } + require_equal( + &format!("duplicate {label} apply"), + &observation, + &client.apply(apply).await?, + )?; + require_equal( + &format!("terminal {label} inspection"), + &observation, + &require_found(label, client.inspect(&observation.unit_id).await?)?, + )?; + let removal = client.remove(remove).await?; + require_equal( + &format!("duplicate {label} removal"), + &removal, + &client.remove(remove).await?, + )?; + require_absent( + &format!("removed {label} inspection"), + client.inspect(&observation.unit_id).await?, + observation.generation, + )?; + Ok((observation, removal)) +} + fn validate_action( action: &RuntimeActionRequest, apply: &RuntimeApplyRequest, diff --git a/src/conformance/profiles.rs b/src/conformance/profiles.rs new file mode 100644 index 0000000..d2991c4 --- /dev/null +++ b/src/conformance/profiles.rs @@ -0,0 +1,395 @@ +use super::{verify_runtime_base, RuntimeBaseConformanceCase, RuntimeBaseConformanceReport}; +use crate::contract::{RuntimeCapabilities, RuntimeFeature}; +use crate::{RuntimeClient, RuntimeError, RuntimeResult}; +use async_trait::async_trait; +use std::collections::{BTreeMap, BTreeSet}; + +/// Composable provider conformance profiles owned by the shared Runtime suite. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum RuntimeConformanceProfile { + Base, + Recovery, + Networking, + Mounts, + Health, + Resources, + Logs, + Exec, + Security, + Outputs, + Evidence, +} + +impl RuntimeConformanceProfile { + pub const fn as_str(self) -> &'static str { + match self { + Self::Base => "base", + Self::Recovery => "recovery", + Self::Networking => "networking", + Self::Mounts => "mounts", + Self::Health => "health", + Self::Resources => "resources", + Self::Logs => "logs", + Self::Exec => "exec", + Self::Security => "security", + Self::Outputs => "outputs", + Self::Evidence => "evidence", + } + } +} + +/// Exact shared case IDs and capability claims required for one profile. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeConformanceProfileRequirements { + pub profile: RuntimeConformanceProfile, + pub case_ids: BTreeSet, + pub capability_claims: BTreeSet, +} + +/// Provider evidence returned for one non-Base profile. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeConformanceProfileEvidence { + pub profile: RuntimeConformanceProfile, + pub case_ids: BTreeSet, + pub capability_claims: BTreeSet, +} + +/// Canonical pre/post provider inventory. Keys can represent provider +/// resources, volumes, ports, mounts, processes, or other retained objects; +/// values are provider-owned stable state digests. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RuntimeConformanceInventory { + pub entries: BTreeMap, +} + +/// Complete report for one profile-driven provider certification run. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeConformanceSuiteReport { + pub base: RuntimeBaseConformanceReport, + pub profiles: Vec, + pub inventory_before: RuntimeConformanceInventory, + pub inventory_after: RuntimeConformanceInventory, +} + +/// Provider integration boundary for destructive, capability-specific +/// fixtures. No method has a success-by-default implementation: a production +/// provider must explicitly declare, execute, clean up, and inventory every +/// activated profile. +#[async_trait] +pub trait RuntimeConformanceFixture: Send + Sync { + fn base_case(&self) -> &RuntimeBaseConformanceCase; + + fn available_profiles(&self) -> BTreeSet; + + async fn inventory(&self) -> RuntimeResult; + + async fn run_profile( + &self, + client: &dyn RuntimeClient, + capabilities: &RuntimeCapabilities, + profile: RuntimeConformanceProfile, + ) -> RuntimeResult; + + async fn cleanup(&self) -> RuntimeResult<()>; +} + +/// Derives mandatory and capability-triggered profiles from source-reported +/// capabilities. Base and Recovery are unconditional. +pub fn required_runtime_profiles( + capabilities: &RuntimeCapabilities, +) -> RuntimeResult> { + capabilities.validate().map_err(RuntimeError::Protocol)?; + let mut profiles = BTreeSet::from([ + RuntimeConformanceProfile::Base, + RuntimeConformanceProfile::Recovery, + ]); + if !capabilities.network_modes.is_empty() { + profiles.insert(RuntimeConformanceProfile::Networking); + } + if !capabilities.mount_kinds.is_empty() { + profiles.insert(RuntimeConformanceProfile::Mounts); + } + if !capabilities.health_check_kinds.is_empty() { + profiles.insert(RuntimeConformanceProfile::Health); + } + if !capabilities.resource_controls.is_empty() { + profiles.insert(RuntimeConformanceProfile::Resources); + } + if capabilities.supports_feature(RuntimeFeature::Logs) { + profiles.insert(RuntimeConformanceProfile::Logs); + } + if capabilities.supports_feature(RuntimeFeature::Exec) { + profiles.insert(RuntimeConformanceProfile::Exec); + } + if !capabilities.isolation_levels.is_empty() + || capabilities.supports_feature(RuntimeFeature::SecretReferences) + { + profiles.insert(RuntimeConformanceProfile::Security); + } + if capabilities.supports_feature(RuntimeFeature::OutputArtifacts) { + profiles.insert(RuntimeConformanceProfile::Outputs); + } + if capabilities.supports_feature(RuntimeFeature::Usage) + || capabilities.supports_feature(RuntimeFeature::Attestation) + { + profiles.insert(RuntimeConformanceProfile::Evidence); + } + Ok(profiles) +} + +/// Returns the stable shared requirements a provider evidence record must +/// cover for the selected profile and capability set. +pub fn runtime_profile_requirements( + capabilities: &RuntimeCapabilities, + profile: RuntimeConformanceProfile, +) -> RuntimeResult { + capabilities.validate().map_err(RuntimeError::Protocol)?; + let mut case_ids = case_ids(profile) + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + if profile == RuntimeConformanceProfile::Security + && capabilities.supports_feature(RuntimeFeature::SecretReferences) + { + case_ids.insert("SECURITY-SECRET-NONDISCLOSURE".into()); + } + if profile == RuntimeConformanceProfile::Evidence + && capabilities.supports_feature(RuntimeFeature::Attestation) + { + case_ids.insert("EVIDENCE-ATTESTATION-VALIDITY".into()); + } + let capability_claims = capability_claims(capabilities, profile); + Ok(RuntimeConformanceProfileRequirements { + profile, + case_ids, + capability_claims, + }) +} + +/// Runs Base internally, activates every required/declared profile, validates +/// typed evidence, always requests provider cleanup, and rejects any inventory +/// delta. +pub async fn verify_runtime_profiles( + client: &dyn RuntimeClient, + fixture: &dyn RuntimeConformanceFixture, +) -> RuntimeResult { + let capabilities = client.capabilities().await?; + capabilities.validate().map_err(RuntimeError::Protocol)?; + let missing_lifecycle = [ + RuntimeFeature::DurableIdentity, + RuntimeFeature::Stop, + RuntimeFeature::Remove, + ] + .into_iter() + .filter(|feature| !capabilities.supports_feature(*feature)) + .map(|feature| format!("feature:{feature:?}")) + .collect::>(); + if !missing_lifecycle.is_empty() { + return Err(RuntimeError::UnsupportedCapabilities(missing_lifecycle)); + } + fixture + .base_case() + .validate() + .map_err(RuntimeError::InvalidRequest)?; + for spec in fixture.base_case().specifications() { + let missing = capabilities + .missing_for(spec) + .map_err(RuntimeError::InvalidRequest)?; + if !missing.is_empty() { + return Err(RuntimeError::UnsupportedCapabilities(missing)); + } + } + + let required = required_runtime_profiles(&capabilities)?; + let mut selected = fixture.available_profiles(); + selected.insert(RuntimeConformanceProfile::Base); + let missing_profiles = required.difference(&selected).copied().collect::>(); + if !missing_profiles.is_empty() { + return Err(RuntimeError::Protocol(format!( + "conformance fixture omits required profiles: {}", + missing_profiles + .iter() + .map(|profile| profile.as_str()) + .collect::>() + .join(", ") + ))); + } + selected.extend(required); + + let inventory_before = fixture.inventory().await?; + let execution: RuntimeResult<( + RuntimeBaseConformanceReport, + Vec, + )> = async { + let base = verify_runtime_base(client, fixture.base_case()).await?; + let mut evidence = vec![base_evidence(&capabilities)?]; + for profile in selected + .iter() + .copied() + .filter(|profile| *profile != RuntimeConformanceProfile::Base) + { + let actual = fixture.run_profile(client, &capabilities, profile).await?; + validate_evidence(&capabilities, profile, &actual)?; + evidence.push(actual); + } + Ok((base, evidence)) + } + .await; + + let cleanup = fixture.cleanup().await; + let inventory_after = fixture.inventory().await; + cleanup?; + let inventory_after = inventory_after?; + if inventory_after != inventory_before { + return Err(RuntimeError::Protocol(format!( + "conformance cleanup changed provider inventory: before={:?}, after={:?}", + inventory_before.entries, inventory_after.entries + ))); + } + let (base, profiles) = execution?; + Ok(RuntimeConformanceSuiteReport { + base, + profiles, + inventory_before, + inventory_after, + }) +} + +fn validate_evidence( + capabilities: &RuntimeCapabilities, + profile: RuntimeConformanceProfile, + evidence: &RuntimeConformanceProfileEvidence, +) -> RuntimeResult<()> { + if evidence.profile != profile { + return Err(RuntimeError::Protocol(format!( + "conformance {} fixture returned {} evidence", + profile.as_str(), + evidence.profile.as_str() + ))); + } + let required = runtime_profile_requirements(capabilities, profile)?; + let missing_cases = required + .case_ids + .difference(&evidence.case_ids) + .cloned() + .collect::>(); + let missing_claims = required + .capability_claims + .difference(&evidence.capability_claims) + .cloned() + .collect::>(); + if !missing_cases.is_empty() || !missing_claims.is_empty() { + return Err(RuntimeError::Protocol(format!( + "conformance {} evidence is incomplete: missing cases {:?}, missing claims {:?}", + profile.as_str(), + missing_cases, + missing_claims + ))); + } + Ok(()) +} + +fn base_evidence( + capabilities: &RuntimeCapabilities, +) -> RuntimeResult { + let required = runtime_profile_requirements(capabilities, RuntimeConformanceProfile::Base)?; + Ok(RuntimeConformanceProfileEvidence { + profile: required.profile, + case_ids: required.case_ids, + capability_claims: required.capability_claims, + }) +} + +fn case_ids(profile: RuntimeConformanceProfile) -> &'static [&'static str] { + match profile { + RuntimeConformanceProfile::Base => &[ + "BASE-TASK-SUCCESS", + "BASE-TASK-FAILURE", + "BASE-TASK-TIMEOUT", + "BASE-SERVICE-LIFECYCLE", + "BASE-EXACT-REPLAY", + "BASE-GENERATION-CONFLICT", + "BASE-TOMBSTONE", + ], + RuntimeConformanceProfile::Recovery => &[ + "RECOVERY-CREATE-BEFORE-ACK", + "RECOVERY-CLIENT-RESTART", + "RECOVERY-PROVIDER-RESTART", + "RECOVERY-EXTERNAL-DELETION", + "RECOVERY-SAME-GENERATION-REPLACEMENT", + "RECOVERY-DUPLICATE-DETECTION", + ], + RuntimeConformanceProfile::Networking => { + &["NETWORK-MODE-BEHAVIOR", "NETWORK-PORT-COLLISION"] + } + RuntimeConformanceProfile::Mounts => &["MOUNT-BEHAVIOR", "MOUNT-ISOLATION-CLEANUP"], + RuntimeConformanceProfile::Health => { + &["HEALTH-PROBE-BEHAVIOR", "HEALTH-THRESHOLD-TRANSITION"] + } + RuntimeConformanceProfile::Resources => { + &["RESOURCE-PROVIDER-INSPECT", "RESOURCE-WORKLOAD-BEHAVIOR"] + } + RuntimeConformanceProfile::Logs => &["LOG-ORDER-FILTER", "LOG-CURSOR-RETENTION"], + RuntimeConformanceProfile::Exec => &["EXEC-STATE-BINDING", "EXEC-TIMEOUT-REPLAY"], + RuntimeConformanceProfile::Security => { + &["SECURITY-DIGEST-NAMESPACE", "SECURITY-METADATA-TAMPER"] + } + RuntimeConformanceProfile::Outputs => &["OUTPUT-EXACT-BOUNDED", "OUTPUT-DIGEST-BINDING"], + RuntimeConformanceProfile::Evidence => &["EVIDENCE-SPEC-BINDING"], + } +} + +fn capability_claims( + capabilities: &RuntimeCapabilities, + profile: RuntimeConformanceProfile, +) -> BTreeSet { + match profile { + RuntimeConformanceProfile::Base => capabilities + .unit_classes + .iter() + .map(|class| format!("unit_class:{class:?}")) + .collect(), + RuntimeConformanceProfile::Recovery => BTreeSet::from(["feature:DurableIdentity".into()]), + RuntimeConformanceProfile::Networking => capabilities + .network_modes + .iter() + .map(|mode| format!("network_mode:{mode:?}")) + .collect(), + RuntimeConformanceProfile::Mounts => capabilities + .mount_kinds + .iter() + .map(|kind| format!("mount_kind:{kind:?}")) + .collect(), + RuntimeConformanceProfile::Health => capabilities + .health_check_kinds + .iter() + .map(|kind| format!("health_check:{kind:?}")) + .collect(), + RuntimeConformanceProfile::Resources => capabilities + .resource_controls + .iter() + .map(|control| format!("resource_control:{control:?}")) + .collect(), + RuntimeConformanceProfile::Logs => BTreeSet::from(["feature:Logs".into()]), + RuntimeConformanceProfile::Exec => BTreeSet::from(["feature:Exec".into()]), + RuntimeConformanceProfile::Security => { + let mut claims = capabilities + .isolation_levels + .iter() + .map(|level| format!("isolation:{level:?}")) + .collect::>(); + claims.insert("feature:DurableIdentity".into()); + if capabilities.supports_feature(RuntimeFeature::SecretReferences) { + claims.insert("feature:SecretReferences".into()); + } + claims + } + RuntimeConformanceProfile::Outputs => BTreeSet::from(["feature:OutputArtifacts".into()]), + RuntimeConformanceProfile::Evidence => [RuntimeFeature::Usage, RuntimeFeature::Attestation] + .into_iter() + .filter(|feature| capabilities.supports_feature(*feature)) + .map(|feature| format!("feature:{feature:?}")) + .collect(), + } +} diff --git a/src/lib.rs b/src/lib.rs index 85eb1dd..8e0f2e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,13 @@ pub mod contract; pub use client::RuntimeClient; pub use clock::{RuntimeClock, SystemRuntimeClock}; -pub use conformance::{verify_runtime_provider, RuntimeConformanceCase, RuntimeConformanceReport}; +pub use conformance::{ + required_runtime_profiles, runtime_profile_requirements, verify_runtime_base, + verify_runtime_profiles, verify_runtime_provider, RuntimeBaseConformanceCase, + RuntimeBaseConformanceReport, RuntimeConformanceCase, RuntimeConformanceFixture, + RuntimeConformanceInventory, RuntimeConformanceProfile, RuntimeConformanceProfileEvidence, + RuntimeConformanceProfileRequirements, RuntimeConformanceReport, RuntimeConformanceSuiteReport, +}; pub use driver::RuntimeDriver; pub use error::{RuntimeError, RuntimeResult}; pub use managed::ManagedRuntimeClient; diff --git a/tests/conformance_profiles.rs b/tests/conformance_profiles.rs new file mode 100644 index 0000000..ef499ab --- /dev/null +++ b/tests/conformance_profiles.rs @@ -0,0 +1,318 @@ +#[path = "process_races/driver.rs"] +mod driver; + +use a3s_runtime::contract::{ + ArtifactRef, HealthCheckKind, IsolationLevel, MountKind, NetworkMode, ResourceLimits, + RestartPolicy, RuntimeActionRequest, RuntimeApplyRequest, RuntimeFeature, RuntimeNetworkSpec, + RuntimeProcessSpec, RuntimeUnitClass, RuntimeUnitSpec, +}; +use a3s_runtime::{ + required_runtime_profiles, runtime_profile_requirements, verify_runtime_profiles, + FileRuntimeStateStore, ManagedRuntimeClient, RuntimeConformanceCase, RuntimeConformanceFixture, + RuntimeConformanceInventory, RuntimeConformanceProfile, RuntimeConformanceProfileEvidence, + RuntimeDriver, RuntimeError, RuntimeResult, +}; +use async_trait::async_trait; +use driver::{ProcessRaceDriver, IMAGE_MEDIA_TYPE}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +fn spec(unit_id: &str, class: RuntimeUnitClass, args: &[&str]) -> RuntimeUnitSpec { + RuntimeUnitSpec { + schema: RuntimeUnitSpec::SCHEMA.into(), + unit_id: unit_id.into(), + generation: 1, + class, + artifact: ArtifactRef { + uri: format!( + "oci://registry.example/a3s/conformance@sha256:{}", + "a".repeat(64) + ), + digest: format!("sha256:{}", "a".repeat(64)), + media_type: IMAGE_MEDIA_TYPE.into(), + }, + process: RuntimeProcessSpec { + command: vec!["/bin/fixture".into()], + args: args.iter().copied().map(Into::into).collect(), + working_directory: None, + environment: BTreeMap::new(), + }, + mounts: Vec::new(), + secrets: Vec::new(), + network: RuntimeNetworkSpec { + mode: NetworkMode::None, + ports: Vec::new(), + }, + resources: ResourceLimits { + cpu_millis: 100, + memory_bytes: 64 * 1024 * 1024, + pids: 32, + ephemeral_storage_bytes: None, + execution_timeout_ms: (class == RuntimeUnitClass::Task).then_some(1_000), + }, + isolation: IsolationLevel::Container, + health: None, + restart: if class == RuntimeUnitClass::Task { + RestartPolicy::Never + } else { + RestartPolicy::Always + }, + outputs: Vec::new(), + semantics_profile_digest: None, + } +} + +fn apply(request_id: &str, spec: RuntimeUnitSpec) -> RuntimeApplyRequest { + RuntimeApplyRequest { + schema: RuntimeApplyRequest::SCHEMA.into(), + request_id: request_id.into(), + deadline_at_ms: None, + spec, + } +} + +fn action(request_id: &str, apply: &RuntimeApplyRequest) -> RuntimeActionRequest { + RuntimeActionRequest { + schema: RuntimeActionRequest::SCHEMA.into(), + request_id: request_id.into(), + unit_id: apply.spec.unit_id.clone(), + generation: apply.spec.generation, + deadline_at_ms: None, + } +} + +fn base_case() -> a3s_runtime::RuntimeBaseConformanceCase { + let task_apply = apply( + "base-task-apply", + spec("base-task", RuntimeUnitClass::Task, &[]), + ); + let service_apply = apply( + "base-service-apply", + spec("base-service", RuntimeUnitClass::Service, &[]), + ); + let task_failure_apply = apply( + "base-failure-apply", + spec("base-failure", RuntimeUnitClass::Task, &["fail"]), + ); + let task_timeout_apply = apply( + "base-timeout-apply", + spec("base-timeout", RuntimeUnitClass::Task, &["timeout"]), + ); + let generation_apply = apply( + "base-generation-apply", + spec("base-generation", RuntimeUnitClass::Service, &[]), + ); + let generation_conflict_apply = apply( + "base-generation-conflict", + spec("base-generation", RuntimeUnitClass::Service, &["changed"]), + ); + a3s_runtime::RuntimeBaseConformanceCase { + lifecycle: RuntimeConformanceCase { + task_remove: action("base-task-remove", &task_apply), + service_stop: action("base-service-stop", &service_apply), + service_remove: action("base-service-remove", &service_apply), + task_apply, + service_apply, + }, + task_failure_remove: action("base-failure-remove", &task_failure_apply), + task_failure_apply, + task_timeout_remove: action("base-timeout-remove", &task_timeout_apply), + task_timeout_apply, + generation_remove: action("base-generation-remove", &generation_apply), + generation_apply, + generation_conflict_apply, + } +} + +struct ProfileFixture { + base: a3s_runtime::RuntimeBaseConformanceCase, + driver: ProcessRaceDriver, + available: BTreeSet, + incomplete: Option, + cleanup_calls: AtomicUsize, +} + +impl ProfileFixture { + fn new(provider_root: &Path) -> Self { + Self { + base: base_case(), + driver: ProcessRaceDriver::new(provider_root), + available: BTreeSet::from([ + RuntimeConformanceProfile::Recovery, + RuntimeConformanceProfile::Networking, + RuntimeConformanceProfile::Resources, + RuntimeConformanceProfile::Security, + ]), + incomplete: None, + cleanup_calls: AtomicUsize::new(0), + } + } + + fn unit_ids(&self) -> Vec { + vec![ + self.base.lifecycle.task_apply.spec.unit_id.clone(), + self.base.lifecycle.service_apply.spec.unit_id.clone(), + self.base.task_failure_apply.spec.unit_id.clone(), + self.base.task_timeout_apply.spec.unit_id.clone(), + self.base.generation_apply.spec.unit_id.clone(), + ] + } +} + +#[async_trait] +impl RuntimeConformanceFixture for ProfileFixture { + fn base_case(&self) -> &a3s_runtime::RuntimeBaseConformanceCase { + &self.base + } + + fn available_profiles(&self) -> BTreeSet { + self.available.clone() + } + + async fn inventory(&self) -> RuntimeResult { + let driver = self.driver.clone(); + let unit_ids = self.unit_ids(); + tokio::task::spawn_blocking(move || { + let mut entries = BTreeMap::new(); + for unit_id in unit_ids { + for resource in driver.inventory(&unit_id)? { + entries.insert( + resource.resource_id, + format!("{}:{:?}", resource.generation, resource.state), + ); + } + } + Ok(RuntimeConformanceInventory { entries }) + }) + .await + .map_err(|error| RuntimeError::Transport(format!("inventory task failed: {error}")))? + } + + async fn run_profile( + &self, + _client: &dyn a3s_runtime::RuntimeClient, + capabilities: &a3s_runtime::contract::RuntimeCapabilities, + profile: RuntimeConformanceProfile, + ) -> RuntimeResult { + let requirements = runtime_profile_requirements(capabilities, profile)?; + let mut evidence = RuntimeConformanceProfileEvidence { + profile, + case_ids: requirements.case_ids, + capability_claims: requirements.capability_claims, + }; + if self.incomplete == Some(profile) { + let _ = evidence.case_ids.pop_first(); + } + Ok(evidence) + } + + async fn cleanup(&self) -> RuntimeResult<()> { + self.cleanup_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +fn client(state_root: &Path, provider_root: &Path) -> ManagedRuntimeClient { + ManagedRuntimeClient::new( + Arc::new(FileRuntimeStateStore::new(state_root)), + Arc::new(ProcessRaceDriver::new(provider_root)), + ) +} + +#[tokio::test] +async fn conf_profile_001_base_and_capability_profiles_run_without_inventory_delta() { + let state = tempfile::tempdir().expect("profile state root"); + let provider = tempfile::tempdir().expect("profile provider root"); + let fixture = ProfileFixture::new(provider.path()); + let report = verify_runtime_profiles(&client(state.path(), provider.path()), &fixture) + .await + .expect("verify profile suite"); + assert_eq!(fixture.cleanup_calls.load(Ordering::SeqCst), 1); + assert_eq!(report.inventory_before, report.inventory_after); + assert_eq!( + report + .profiles + .iter() + .map(|evidence| evidence.profile) + .collect::>(), + BTreeSet::from([ + RuntimeConformanceProfile::Base, + RuntimeConformanceProfile::Recovery, + RuntimeConformanceProfile::Networking, + RuntimeConformanceProfile::Resources, + RuntimeConformanceProfile::Security, + ]) + ); +} + +#[tokio::test] +async fn conf_profile_002_missing_mandatory_profile_fails_before_provider_work() { + let state = tempfile::tempdir().expect("missing-profile state root"); + let provider = tempfile::tempdir().expect("missing-profile provider root"); + let mut fixture = ProfileFixture::new(provider.path()); + fixture + .available + .remove(&RuntimeConformanceProfile::Recovery); + assert!(matches!( + verify_runtime_profiles(&client(state.path(), provider.path()), &fixture).await, + Err(RuntimeError::Protocol(message)) if message.contains("recovery") + )); + assert_eq!(fixture.cleanup_calls.load(Ordering::SeqCst), 0); + assert!(fixture + .driver + .inventory("base-task") + .expect("missing-profile inventory") + .is_empty()); +} + +#[tokio::test] +async fn conf_profile_003_incomplete_evidence_fails_and_still_cleans_up() { + let state = tempfile::tempdir().expect("incomplete-profile state root"); + let provider = tempfile::tempdir().expect("incomplete-profile provider root"); + let mut fixture = ProfileFixture::new(provider.path()); + fixture.incomplete = Some(RuntimeConformanceProfile::Recovery); + assert!(matches!( + verify_runtime_profiles(&client(state.path(), provider.path()), &fixture).await, + Err(RuntimeError::Protocol(message)) if message.contains("evidence is incomplete") + )); + assert_eq!(fixture.cleanup_calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn conf_profile_004_all_advertised_optional_families_activate() { + let provider = tempfile::tempdir().expect("activation provider root"); + let driver = ProcessRaceDriver::new(provider.path()); + let mut capabilities = driver.capabilities().await.expect("driver capabilities"); + capabilities.mount_kinds = vec![MountKind::Volume]; + capabilities.health_check_kinds = vec![HealthCheckKind::Command]; + capabilities + .isolation_levels + .push(IsolationLevel::Confidential); + capabilities.features.extend([ + RuntimeFeature::Logs, + RuntimeFeature::Exec, + RuntimeFeature::SecretReferences, + RuntimeFeature::OutputArtifacts, + RuntimeFeature::Usage, + RuntimeFeature::Attestation, + ]); + let profiles = required_runtime_profiles(&capabilities).expect("derive profiles"); + assert_eq!( + profiles, + BTreeSet::from([ + RuntimeConformanceProfile::Base, + RuntimeConformanceProfile::Recovery, + RuntimeConformanceProfile::Networking, + RuntimeConformanceProfile::Mounts, + RuntimeConformanceProfile::Health, + RuntimeConformanceProfile::Resources, + RuntimeConformanceProfile::Logs, + RuntimeConformanceProfile::Exec, + RuntimeConformanceProfile::Security, + RuntimeConformanceProfile::Outputs, + RuntimeConformanceProfile::Evidence, + ]) + ); +} diff --git a/tests/process_races/driver.rs b/tests/process_races/driver.rs index 644a284..7cd6a6b 100644 --- a/tests/process_races/driver.rs +++ b/tests/process_races/driver.rs @@ -1,8 +1,8 @@ use a3s_runtime::contract::{ IsolationLevel, NetworkMode, ResourceControl, RuntimeActionRequest, RuntimeCapabilities, - RuntimeExecRequest, RuntimeExecResult, RuntimeFeature, RuntimeInspection, RuntimeLogChunk, - RuntimeLogQuery, RuntimeObservation, RuntimeRemoval, RuntimeUnitClass, RuntimeUnitSpec, - RuntimeUnitState, + RuntimeExecRequest, RuntimeExecResult, RuntimeFailure, RuntimeFeature, RuntimeInspection, + RuntimeLogChunk, RuntimeLogQuery, RuntimeObservation, RuntimeRemoval, RuntimeUnitClass, + RuntimeUnitSpec, RuntimeUnitState, }; use a3s_runtime::{ProviderId, RuntimeDriver, RuntimeError, RuntimeResult, RuntimeUnitRecord}; use async_trait::async_trait; @@ -97,20 +97,34 @@ impl ProcessRaceDriver { .filter(|resource| resource.generation == spec.generation) .collect::>(); - let mut resource = current_resources.first().map_or_else( - || ProviderResource { + let target_state = match spec.class { + RuntimeUnitClass::Task + if spec + .process + .args + .iter() + .any(|argument| matches!(argument.as_str(), "fail" | "timeout")) => + { + RuntimeUnitState::Failed + } + RuntimeUnitClass::Task => RuntimeUnitState::Succeeded, + RuntimeUnitClass::Service => RuntimeUnitState::Running, + }; + let mut resource = current_resources + .first() + .copied() + .cloned() + .unwrap_or_else(|| ProviderResource { resource_id: format!("process/{}/g{}", spec.unit_id, spec.generation), unit_id: spec.unit_id.clone(), generation: spec.generation, - state: RuntimeUnitState::Running, + state: target_state, observed_at_ms: current.observed_at_ms.saturating_add(1), started_at_ms: current.observed_at_ms.saturating_add(1), apply_dispatches: 0, - }, - |resource| (*resource).clone(), - ); + }); resource.apply_dispatches = resource.apply_dispatches.saturating_add(1); - resource.state = RuntimeUnitState::Running; + resource.state = target_state; resource.observed_at_ms = resource .observed_at_ms .max(current.observed_at_ms.saturating_add(1)); @@ -231,7 +245,7 @@ impl RuntimeDriver for ProcessRaceDriver { schema: RuntimeCapabilities::SCHEMA.into(), provider_id: self.provider_id.clone(), provider_build: PROVIDER_BUILD.into(), - unit_classes: vec![RuntimeUnitClass::Service], + unit_classes: vec![RuntimeUnitClass::Task, RuntimeUnitClass::Service], artifact_media_types: vec![IMAGE_MEDIA_TYPE.into()], isolation_levels: vec![IsolationLevel::Container], network_modes: vec![NetworkMode::None], @@ -241,6 +255,7 @@ impl RuntimeDriver for ProcessRaceDriver { ResourceControl::Cpu, ResourceControl::Memory, ResourceControl::Pids, + ResourceControl::ExecutionTimeout, ], features: vec![ RuntimeFeature::DurableIdentity, @@ -398,7 +413,11 @@ fn observation_from_resource( .then_some(observation.observed_at_ms); observation.health = None; observation.outputs.clear(); - observation.failure = None; + observation.failure = (resource.state == RuntimeUnitState::Failed).then(|| RuntimeFailure { + code: "fixture_failure".into(), + message: "process-race fixture failed as requested".into(), + retryable: false, + }); observation } From 2fd08d19e08e78b0f0b2104d36d79b547ef86262 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 15:13:40 +0800 Subject: [PATCH 27/32] test: scope duplicate injection to race suite --- tests/process_races.rs | 24 +++++++++++++++++++++--- tests/process_races/driver.rs | 13 ------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/tests/process_races.rs b/tests/process_races.rs index 1eb9b3c..70a33f0 100644 --- a/tests/process_races.rs +++ b/tests/process_races.rs @@ -11,6 +11,7 @@ use a3s_runtime::{ RuntimeStateStore, }; use driver::{ProcessRaceDriver, ProviderResource, IMAGE_MEDIA_TYPE}; +use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process::{Child, Command}; @@ -274,6 +275,25 @@ fn assert_single_generation(resources: &[ProviderResource], generation: u64) { assert_eq!(resources[0].generation, generation); } +fn inject_duplicate(provider_root: &Path, unit_id: &str, generation: u64) { + let driver = ProcessRaceDriver::new(provider_root); + let source = driver + .inventory(unit_id) + .expect("load duplicate source") + .into_iter() + .find(|resource| resource.generation == generation) + .expect("duplicate source is absent"); + let mut duplicate = source; + duplicate.resource_id = format!("{}/duplicate", duplicate.resource_id); + let key = format!("{:x}", Sha256::digest(duplicate.resource_id.as_bytes())); + let path = provider_root.join("resources").join(format!("{key}.json")); + std::fs::write( + path, + serde_json::to_vec(&duplicate).expect("encode duplicate provider resource"), + ) + .expect("inject duplicate provider resource"); +} + #[test] fn race_gen_001_concurrent_generations_converge_to_one_newest_resource() { let state = tempfile::tempdir().expect("generation-race state root"); @@ -482,9 +502,7 @@ fn race_inventory_001_duplicate_resources_fail_closed_before_provider_mutation() .block_on(client.apply(&apply_request("duplicate-one", unit_id, 1))) .expect("prepare duplicate provider fixture"); let process_driver = ProcessRaceDriver::new(provider.path()); - process_driver - .inject_duplicate(unit_id, 1) - .expect("inject duplicate provider resource"); + inject_duplicate(provider.path(), unit_id, 1); assert_eq!( process_driver .inventory(unit_id) diff --git a/tests/process_races/driver.rs b/tests/process_races/driver.rs index 7cd6a6b..23bc47c 100644 --- a/tests/process_races/driver.rs +++ b/tests/process_races/driver.rs @@ -50,19 +50,6 @@ impl ProcessRaceDriver { self.with_lock(|root| inventory_unlocked(root, unit_id)) } - pub fn inject_duplicate(&self, unit_id: &str, generation: u64) -> RuntimeResult<()> { - self.with_lock(|root| { - let resources = inventory_unlocked(root, unit_id)?; - let source = resources - .iter() - .find(|resource| resource.generation == generation) - .ok_or_else(|| RuntimeError::Protocol("duplicate source is absent".into()))?; - let mut duplicate = source.clone(); - duplicate.resource_id = format!("{}/duplicate", source.resource_id); - write_resource(root, &duplicate) - }) - } - fn with_lock(&self, operation: impl FnOnce(&Path) -> RuntimeResult) -> RuntimeResult { ensure_provider_root(&self.root)?; let lock_path = self.root.join("provider.lock"); From cf636273d9329891f0f8c070a08938f4ef44262e Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 15:16:00 +0800 Subject: [PATCH 28/32] docs: explain profile-driven conformance --- README.md | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index f497ac6..bc6632f 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,9 @@ their owning applications. cross-process locks without following symbolic-link state boundaries - **Logs and Exec**: Expose generation-bound log and exec surfaces only when a provider reports the corresponding capability -- **Conformance Suite**: Exercise the common Task and Service lifecycle against - provider-owned disposable resources +- **Capability-Driven Conformance**: Always run complete Base and Recovery + profiles, automatically activate advertised optional profiles, and reject + missing fixtures, incomplete evidence, or provider inventory leaks ## Runtime Model @@ -182,29 +183,32 @@ stay behind the driver. ## Conformance -Provider repositories should run `verify_runtime_provider` against real, -disposable infrastructure: +Production provider repositories should implement `RuntimeConformanceFixture` +and run `verify_runtime_profiles` against real, disposable infrastructure: ```rust,ignore -use a3s_runtime::{verify_runtime_provider, RuntimeConformanceCase}; - -let case = RuntimeConformanceCase { - task_apply, - task_remove, - service_apply, - service_stop, - service_remove, -}; - -let report = verify_runtime_provider(client.as_ref(), &case).await?; -assert!(report.task.converges(&task_spec)); -assert!(report.service.converges(&service_spec)); +use a3s_runtime::{verify_runtime_profiles, RuntimeConformanceFixture}; + +let fixture: &dyn RuntimeConformanceFixture = provider_fixture; +let report = verify_runtime_profiles(client.as_ref(), fixture).await?; +assert_eq!(report.inventory_before, report.inventory_after); ``` -The shared suite validates capability matching, exact apply replay, inspection, -stop replay, removal replay, and generation-aware absence for both lifecycle -classes. Provider repositories remain responsible for crash injection, -reconstruction, provider-specific security, and resource-leak tests. +The mandatory Base profile covers successful, failed, and timed-out Tasks; +Service apply, inspect, stop, and removal; exact replay; generation conflicts; +and tombstones. Recovery is mandatory for every production provider. +Networking, Mounts, Health, Resources, Logs, Exec, Security, Outputs, and +Evidence activate from reported capabilities. A fixture must return the shared +stable case IDs and capability claims for every activated profile, perform +cleanup even after a failed profile, and prove that its canonical provider +inventory returned to the pre-run baseline. + +`verify_runtime_provider` remains available as the lower-level successful Task +and Service lifecycle check. It is not, by itself, production certification. +Provider-specific fixtures still own real daemon restart, external deletion, +security, resource-behavior, and destructive cleanup mechanics; the shared +harness owns activation, required case/claim coverage, Base semantics, and the +zero-inventory-delta oracle. See the [deep test plan](docs/deep-test-plan.md) for the full contract, durability, real-provider, fault, performance, soak, and A3S OS release gates. From da7514c7a87aa14b448b90d92acbca290cb30075 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 15:35:37 +0800 Subject: [PATCH 29/32] test: require behavior-level provider evidence --- README.md | 9 +++ src/conformance/profiles.rs | 140 ++++++++++++++++++++++++++++++---- tests/conformance_profiles.rs | 112 ++++++++++++++++++++++++++- 3 files changed, 244 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index bc6632f..71e4552 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,15 @@ stable case IDs and capability claims for every activated profile, perform cleanup even after a failed profile, and prove that its canonical provider inventory returned to the pre-run baseline. +Profile requirements expand to one case ID per advertised behavior rather than +accepting a generic family-level claim. For example, every reported network +mode, mount kind, health probe, and resource control activates its own +configuration and behavioral cases. `NetworkMode::Service` activates both TCP +and UDP because the current protocol has no narrower transport-protocol +capability. Logs separately require filtering, total order, cursor resume, +same-timestamp handling, limits, explicit rotation gaps, terminal retention, +and bounded large records. + `verify_runtime_provider` remains available as the lower-level successful Task and Service lifecycle check. It is not, by itself, production certification. Provider-specific fixtures still own real daemon restart, external deletion, diff --git a/src/conformance/profiles.rs b/src/conformance/profiles.rs index d2991c4..a85e7fa 100644 --- a/src/conformance/profiles.rs +++ b/src/conformance/profiles.rs @@ -1,5 +1,7 @@ use super::{verify_runtime_base, RuntimeBaseConformanceCase, RuntimeBaseConformanceReport}; -use crate::contract::{RuntimeCapabilities, RuntimeFeature}; +use crate::contract::{ + HealthCheckKind, MountKind, NetworkMode, ResourceControl, RuntimeCapabilities, RuntimeFeature, +}; use crate::{RuntimeClient, RuntimeError, RuntimeResult}; use async_trait::async_trait; use std::collections::{BTreeMap, BTreeSet}; @@ -144,11 +146,12 @@ pub fn runtime_profile_requirements( profile: RuntimeConformanceProfile, ) -> RuntimeResult { capabilities.validate().map_err(RuntimeError::Protocol)?; - let mut case_ids = case_ids(profile) + let mut case_ids = base_case_ids(profile) .iter() .copied() .map(str::to_owned) .collect::>(); + extend_capability_case_ids(capabilities, profile, &mut case_ids); if profile == RuntimeConformanceProfile::Security && capabilities.supports_feature(RuntimeFeature::SecretReferences) { @@ -159,6 +162,11 @@ pub fn runtime_profile_requirements( { case_ids.insert("EVIDENCE-ATTESTATION-VALIDITY".into()); } + if profile == RuntimeConformanceProfile::Evidence + && capabilities.supports_feature(RuntimeFeature::Usage) + { + case_ids.insert("EVIDENCE-USAGE-VALIDITY".into()); + } let capability_claims = capability_claims(capabilities, profile); Ok(RuntimeConformanceProfileRequirements { profile, @@ -301,7 +309,7 @@ fn base_evidence( }) } -fn case_ids(profile: RuntimeConformanceProfile) -> &'static [&'static str] { +fn base_case_ids(profile: RuntimeConformanceProfile) -> &'static [&'static str] { match profile { RuntimeConformanceProfile::Base => &[ "BASE-TASK-SUCCESS", @@ -320,23 +328,127 @@ fn case_ids(profile: RuntimeConformanceProfile) -> &'static [&'static str] { "RECOVERY-SAME-GENERATION-REPLACEMENT", "RECOVERY-DUPLICATE-DETECTION", ], + RuntimeConformanceProfile::Networking => &[], + RuntimeConformanceProfile::Mounts => &["MOUNT-READ-ONLY", "MOUNT-CLEANUP"], + RuntimeConformanceProfile::Health => &[ + "HEALTH-THRESHOLD-TRANSITION", + "HEALTH-PROBE-TIMEOUT", + "HEALTH-START-PERIOD", + "HEALTH-UNHEALTHY-EXIT", + ], + RuntimeConformanceProfile::Resources => &[], + RuntimeConformanceProfile::Logs => &[ + "LOG-STREAM-FILTER", + "LOG-TOTAL-ORDER", + "LOG-CURSOR-RESUME", + "LOG-SAME-TIMESTAMP", + "LOG-LIMIT", + "LOG-ROTATION-GAP", + "LOG-RETENTION", + "LOG-LARGE-RECORD", + ], + RuntimeConformanceProfile::Exec => &[ + "EXEC-STATE-POLICY", + "EXEC-TIMEOUT-REPLAY", + "EXEC-EXIT-CODE", + "EXEC-OUTPUT-BOUNDS", + "EXEC-OUTPUT-TRUNCATION", + "EXEC-IDENTITY-GENERATION-BINDING", + ], + RuntimeConformanceProfile::Security => &[ + "SECURITY-DIGEST-PINNING", + "SECURITY-METADATA-TAMPER", + "SECURITY-NAMESPACE-SEPARATION", + "SECURITY-LEAST-PRIVILEGE", + "SECURITY-HOSTILE-INPUT", + ], + RuntimeConformanceProfile::Outputs => &["OUTPUT-EXACT-BOUNDED", "OUTPUT-DIGEST-BINDING"], + RuntimeConformanceProfile::Evidence => &[ + "EVIDENCE-SPEC-BINDING", + "EVIDENCE-SEMANTICS-PROFILE-BINDING", + ], + } +} + +fn extend_capability_case_ids( + capabilities: &RuntimeCapabilities, + profile: RuntimeConformanceProfile, + case_ids: &mut BTreeSet, +) { + match profile { RuntimeConformanceProfile::Networking => { - &["NETWORK-MODE-BEHAVIOR", "NETWORK-PORT-COLLISION"] + for mode in &capabilities.network_modes { + match mode { + NetworkMode::None => { + case_ids.insert("NETWORK-MODE-NONE".into()); + case_ids.insert("NETWORK-OUTBOUND-DENIED".into()); + } + NetworkMode::Outbound => { + case_ids.insert("NETWORK-MODE-OUTBOUND".into()); + case_ids.insert("NETWORK-OUTBOUND-ALLOWED".into()); + } + NetworkMode::Service => { + case_ids.insert("NETWORK-MODE-SERVICE".into()); + case_ids.insert("NETWORK-PROTOCOL-TCP".into()); + case_ids.insert("NETWORK-PROTOCOL-UDP".into()); + case_ids.insert("NETWORK-LOOPBACK-PUBLICATION".into()); + case_ids.insert("NETWORK-PORT-COLLISION".into()); + } + } + } + } + RuntimeConformanceProfile::Mounts => { + for kind in &capabilities.mount_kinds { + case_ids.insert( + match kind { + MountKind::Artifact => "MOUNT-ARTIFACT-BEHAVIOR", + MountKind::Volume => "MOUNT-VOLUME-PERSISTENCE", + MountKind::Tmpfs => "MOUNT-TMPFS-ISOLATION", + } + .into(), + ); + } } - RuntimeConformanceProfile::Mounts => &["MOUNT-BEHAVIOR", "MOUNT-ISOLATION-CLEANUP"], RuntimeConformanceProfile::Health => { - &["HEALTH-PROBE-BEHAVIOR", "HEALTH-THRESHOLD-TRANSITION"] + for kind in &capabilities.health_check_kinds { + case_ids.insert( + match kind { + HealthCheckKind::Http => "HEALTH-PROBE-HTTP", + HealthCheckKind::Tcp => "HEALTH-PROBE-TCP", + HealthCheckKind::Command => "HEALTH-PROBE-COMMAND", + } + .into(), + ); + } } RuntimeConformanceProfile::Resources => { - &["RESOURCE-PROVIDER-INSPECT", "RESOURCE-WORKLOAD-BEHAVIOR"] - } - RuntimeConformanceProfile::Logs => &["LOG-ORDER-FILTER", "LOG-CURSOR-RETENTION"], - RuntimeConformanceProfile::Exec => &["EXEC-STATE-BINDING", "EXEC-TIMEOUT-REPLAY"], - RuntimeConformanceProfile::Security => { - &["SECURITY-DIGEST-NAMESPACE", "SECURITY-METADATA-TAMPER"] + for control in &capabilities.resource_controls { + let (configuration, behavior) = match control { + ResourceControl::Cpu => ("RESOURCE-CPU-CONFIG", "RESOURCE-CPU-BEHAVIOR"), + ResourceControl::Memory => { + ("RESOURCE-MEMORY-CONFIG", "RESOURCE-MEMORY-BEHAVIOR") + } + ResourceControl::Pids => ("RESOURCE-PIDS-CONFIG", "RESOURCE-PIDS-BEHAVIOR"), + ResourceControl::EphemeralStorage => ( + "RESOURCE-EPHEMERAL-STORAGE-CONFIG", + "RESOURCE-EPHEMERAL-STORAGE-BEHAVIOR", + ), + ResourceControl::ExecutionTimeout => ( + "RESOURCE-EXECUTION-TIMEOUT-CONFIG", + "RESOURCE-EXECUTION-TIMEOUT-BEHAVIOR", + ), + }; + case_ids.insert(configuration.into()); + case_ids.insert(behavior.into()); + } } - RuntimeConformanceProfile::Outputs => &["OUTPUT-EXACT-BOUNDED", "OUTPUT-DIGEST-BINDING"], - RuntimeConformanceProfile::Evidence => &["EVIDENCE-SPEC-BINDING"], + RuntimeConformanceProfile::Base + | RuntimeConformanceProfile::Recovery + | RuntimeConformanceProfile::Logs + | RuntimeConformanceProfile::Exec + | RuntimeConformanceProfile::Security + | RuntimeConformanceProfile::Outputs + | RuntimeConformanceProfile::Evidence => {} } } diff --git a/tests/conformance_profiles.rs b/tests/conformance_profiles.rs index ef499ab..4238b3a 100644 --- a/tests/conformance_profiles.rs +++ b/tests/conformance_profiles.rs @@ -2,9 +2,9 @@ mod driver; use a3s_runtime::contract::{ - ArtifactRef, HealthCheckKind, IsolationLevel, MountKind, NetworkMode, ResourceLimits, - RestartPolicy, RuntimeActionRequest, RuntimeApplyRequest, RuntimeFeature, RuntimeNetworkSpec, - RuntimeProcessSpec, RuntimeUnitClass, RuntimeUnitSpec, + ArtifactRef, HealthCheckKind, IsolationLevel, MountKind, NetworkMode, ResourceControl, + ResourceLimits, RestartPolicy, RuntimeActionRequest, RuntimeApplyRequest, RuntimeFeature, + RuntimeNetworkSpec, RuntimeProcessSpec, RuntimeUnitClass, RuntimeUnitSpec, }; use a3s_runtime::{ required_runtime_profiles, runtime_profile_requirements, verify_runtime_profiles, @@ -316,3 +316,109 @@ async fn conf_profile_004_all_advertised_optional_families_activate() { ]) ); } + +#[tokio::test] +async fn conf_profile_005_requirements_expand_every_advertised_behavior() { + let provider = tempfile::tempdir().expect("requirements provider root"); + let driver = ProcessRaceDriver::new(provider.path()); + let mut capabilities = driver.capabilities().await.expect("driver capabilities"); + capabilities.network_modes = vec![ + NetworkMode::None, + NetworkMode::Outbound, + NetworkMode::Service, + ]; + capabilities.mount_kinds = vec![MountKind::Volume, MountKind::Tmpfs]; + capabilities.health_check_kinds = vec![ + HealthCheckKind::Http, + HealthCheckKind::Tcp, + HealthCheckKind::Command, + ]; + capabilities.resource_controls = vec![ + ResourceControl::Cpu, + ResourceControl::Memory, + ResourceControl::Pids, + ResourceControl::ExecutionTimeout, + ]; + capabilities.features.extend([ + RuntimeFeature::Logs, + RuntimeFeature::Usage, + RuntimeFeature::Attestation, + ]); + + let networking = + runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Networking) + .expect("networking requirements"); + assert_eq!( + networking.case_ids, + BTreeSet::from([ + "NETWORK-LOOPBACK-PUBLICATION".into(), + "NETWORK-MODE-NONE".into(), + "NETWORK-MODE-OUTBOUND".into(), + "NETWORK-MODE-SERVICE".into(), + "NETWORK-OUTBOUND-ALLOWED".into(), + "NETWORK-OUTBOUND-DENIED".into(), + "NETWORK-PORT-COLLISION".into(), + "NETWORK-PROTOCOL-TCP".into(), + "NETWORK-PROTOCOL-UDP".into(), + ]) + ); + + let mounts = runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Mounts) + .expect("mount requirements"); + assert_eq!( + mounts.case_ids, + BTreeSet::from([ + "MOUNT-CLEANUP".into(), + "MOUNT-READ-ONLY".into(), + "MOUNT-TMPFS-ISOLATION".into(), + "MOUNT-VOLUME-PERSISTENCE".into(), + ]) + ); + + let health = runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Health) + .expect("health requirements"); + assert_eq!( + health.case_ids, + BTreeSet::from([ + "HEALTH-PROBE-COMMAND".into(), + "HEALTH-PROBE-HTTP".into(), + "HEALTH-PROBE-TCP".into(), + "HEALTH-PROBE-TIMEOUT".into(), + "HEALTH-START-PERIOD".into(), + "HEALTH-THRESHOLD-TRANSITION".into(), + "HEALTH-UNHEALTHY-EXIT".into(), + ]) + ); + + let resources = + runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Resources) + .expect("resource requirements"); + assert_eq!(resources.case_ids.len(), 8); + for required in [ + "RESOURCE-CPU-CONFIG", + "RESOURCE-CPU-BEHAVIOR", + "RESOURCE-MEMORY-CONFIG", + "RESOURCE-MEMORY-BEHAVIOR", + "RESOURCE-PIDS-CONFIG", + "RESOURCE-PIDS-BEHAVIOR", + "RESOURCE-EXECUTION-TIMEOUT-CONFIG", + "RESOURCE-EXECUTION-TIMEOUT-BEHAVIOR", + ] { + assert!(resources.case_ids.contains(required)); + } + + let logs = runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Logs) + .expect("log requirements"); + assert_eq!(logs.case_ids.len(), 8); + assert!(logs.case_ids.contains("LOG-SAME-TIMESTAMP")); + assert!(logs.case_ids.contains("LOG-ROTATION-GAP")); + assert!(logs.case_ids.contains("LOG-LARGE-RECORD")); + + let evidence = runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Evidence) + .expect("evidence requirements"); + assert!(evidence.case_ids.contains("EVIDENCE-USAGE-VALIDITY")); + assert!(evidence.case_ids.contains("EVIDENCE-ATTESTATION-VALIDITY")); + assert!(evidence + .case_ids + .contains("EVIDENCE-SEMANTICS-PROFILE-BINDING")); +} From f1161b2e8a14c73b22af6c6fc10ebc8814631f41 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 15:55:26 +0800 Subject: [PATCH 30/32] docs: make Docker certification gate explicit --- docs/deep-test-plan.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/deep-test-plan.md b/docs/deep-test-plan.md index 26a0736..da0d572 100644 --- a/docs/deep-test-plan.md +++ b/docs/deep-test-plan.md @@ -159,9 +159,16 @@ The real Docker gate runs from the A3S Cloud repository root: ```text A3S_CLOUD_TEST_DOCKER=1 cargo test -p a3s-cloud-node-agent \ - --test docker_conformance -- --nocapture --test-threads=1 + --test docker_conformance real_docker_passes_all_advertised_runtime_profiles \ + -- --ignored --exact --nocapture --test-threads=1 ``` +The Docker test is explicitly ignored in ordinary workspace runs. Its +dedicated job must select the ignored test by exact name and fail unless the +Docker enable flag, isolated provider restart target, and provider socket are +present. This prevents an unavailable provider from being counted as a passing +certification. + The Runtime crate must declare a minimum supported Rust version before release; L0 tests that version and current stable on the production Linux architectures. From e68054612a1d488e1bcc295f92467fdc655f68c2 Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 22:06:02 +0800 Subject: [PATCH 31/32] fix(exec): forward durable effective deadline --- src/managed.rs | 4 ++- tests/general_runtime.rs | 67 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/managed.rs b/src/managed.rs index 11eeac4..85b7c36 100644 --- a/src/managed.rs +++ b/src/managed.rs @@ -552,11 +552,13 @@ impl RuntimeClient for ManagedRuntimeClient { })?; let deadline_at_ms = Some(deadline_at_ms); self.check_deadline(deadline_at_ms)?; + let mut provider_request = request.clone(); + provider_request.deadline_at_ms = deadline_at_ms; let result = self .bounded( deadline_at_ms, "provider exec", - self.driver.exec(&reservation.record, request), + self.driver.exec(&reservation.record, &provider_request), ) .await?; result.validate().map_err(RuntimeError::Protocol)?; diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index 574b044..eb01ded 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -61,6 +61,7 @@ struct TestDriver { remove_calls: AtomicUsize, exec_calls: AtomicUsize, exec_effects: Mutex>, + last_exec_request: Mutex>, exec_stdout_bytes: AtomicUsize, exec_truncated: AtomicBool, fail_next_apply: AtomicBool, @@ -90,6 +91,7 @@ impl TestDriver { remove_calls: AtomicUsize::new(0), exec_calls: AtomicUsize::new(0), exec_effects: Mutex::new(BTreeMap::new()), + last_exec_request: Mutex::new(None), exec_stdout_bytes: AtomicUsize::new(0), exec_truncated: AtomicBool::new(false), fail_next_apply: AtomicBool::new(false), @@ -495,6 +497,7 @@ impl RuntimeDriver for TestDriver { request: &RuntimeExecRequest, ) -> RuntimeResult { self.exec_calls.fetch_add(1, Ordering::SeqCst); + *self.last_exec_request.lock().unwrap() = Some(request.clone()); if self.hang_exec.load(Ordering::SeqCst) { return std::future::pending::>().await; } @@ -1407,7 +1410,8 @@ async fn expired_pending_request_remains_pending_and_is_not_redispatched() { let clock = Arc::new(ManualClock::new(NOW)); let client = ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); let mut request = apply("pending-expiry", service("pending-expiry", 1)); - request.deadline_at_ms = Some(NOW + 10); + let deadline = NOW + 1_000; + request.deadline_at_ms = Some(deadline); driver.hang_apply.store(true, Ordering::SeqCst); assert!(matches!( @@ -1424,7 +1428,7 @@ async fn expired_pending_request_remains_pending_and_is_not_redispatched() { a3s_runtime::RuntimeRequestState::Pending ); - clock.set(NOW + 10); + clock.set(deadline); driver.hang_apply.store(false, Ordering::SeqCst); assert!(matches!( client.apply(&request).await, @@ -1836,6 +1840,65 @@ async fn exec_uses_the_smaller_relative_or_absolute_deadline() { assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); } +#[tokio::test] +async fn lc_exec_003_provider_receives_the_durable_effective_deadline() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let clock = Arc::new(ManualClock::new(NOW)); + let client = ManagedRuntimeClient::with_clock(store, driver.clone(), clock.clone()); + client + .apply(&apply( + "exec-provider-deadline-apply", + service("exec-provider-deadline", 1), + )) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-provider-deadline-request".into(), + unit_id: "exec-provider-deadline".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }; + driver + .fail_next_exec_after_effect + .store(true, Ordering::SeqCst); + + assert!(matches!( + client.exec(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("ambiguous") + )); + assert_eq!( + driver + .last_exec_request + .lock() + .unwrap() + .as_ref() + .unwrap() + .deadline_at_ms, + Some(NOW + 1_000) + ); + + clock.set(NOW + 200); + let replayed = client.exec(&request).await.unwrap(); + assert_eq!(replayed.request_id, request.request_id); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 2); + assert_eq!(driver.exec_effect_count(), 1); + assert_eq!( + driver + .last_exec_request + .lock() + .unwrap() + .as_ref() + .unwrap() + .deadline_at_ms, + Some(NOW + 1_000) + ); +} + #[tokio::test] async fn lc_exec_001_pending_replay_cannot_extend_original_relative_deadline() { let directory = tempfile::tempdir().unwrap(); From 91a1b4c67ef2691915f2409a8cf835f7534020ff Mon Sep 17 00:00:00 2001 From: RoyLin Date: Fri, 17 Jul 2026 22:18:46 +0800 Subject: [PATCH 32/32] docs(exec): define durable provider deadline --- README.md | 6 +++++- .../0002-complete-protocol-and-operation-semantics.md | 5 ++++- src/driver.rs | 9 +++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 71e4552..d7304c2 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,11 @@ An exact retry returns or reconstructs the same logical result. Reusing a request ID with different content fails with `RequestConflict`. A completed receipt remains replayable after its original deadline and after later lifecycle operations; an expired pending request is not redispatched. A -deadline is checked independently before new provider work. +deadline is checked independently before new provider work. On the first exec +reservation, Runtime persists the smaller of `started_at + timeout_ms` and the +optional caller deadline. `RuntimeDriver::exec` receives that effective +absolute deadline in `deadline_at_ms`, and every pending replay receives the +same value, so retrying cannot restart or extend the execution window. ## Capabilities diff --git a/docs/adr/0002-complete-protocol-and-operation-semantics.md b/docs/adr/0002-complete-protocol-and-operation-semantics.md index 0fa6185..424f239 100644 --- a/docs/adr/0002-complete-protocol-and-operation-semantics.md +++ b/docs/adr/0002-complete-protocol-and-operation-semantics.md @@ -216,7 +216,10 @@ deadline and is never redispatched after that deadline expires. The request receipt stores the effective absolute deadline captured on first reservation. For Exec this is the smaller of the first attempt's relative timeout and optional absolute deadline; a retry cannot restart that relative -timeout window. +timeout window. Before provider dispatch, `ManagedRuntimeClient` replaces the +driver-bound exec request's optional caller deadline with that persisted +effective absolute deadline. The driver therefore receives the same non-null +`deadline_at_ms` on the first dispatch and every pending replay. Drivers may enforce a shorter provider-specific timeout. They must never extend the caller deadline. Exec uses the smaller of its relative `timeout_ms` and an diff --git a/src/driver.rs b/src/driver.rs index b66a94e..cc0fa30 100644 --- a/src/driver.rs +++ b/src/driver.rs @@ -47,6 +47,15 @@ pub trait RuntimeDriver: Send + Sync { query: &RuntimeLogQuery, ) -> RuntimeResult>; + /// Executes one durably identified request within its original budget. + /// + /// `ManagedRuntimeClient` always supplies `request.deadline_at_ms` as the + /// effective absolute deadline captured by the first reservation: the + /// smaller of that attempt's `timeout_ms` window and any caller-provided + /// absolute deadline. A pending replay receives the same persisted value, + /// so a driver must not restart or extend the execution window. Drivers may + /// enforce a shorter provider-specific timeout and must deduplicate or + /// reattach the stable request ID after an ambiguous result. async fn exec( &self, unit: &RuntimeUnitRecord,