diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e56fc57 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,55 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: runtime-ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + +jobs: + linux: + name: Linux / Rust ${{ matrix.toolchain }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + toolchain: + - "1.85.0" + - stable + + steps: + - name: Check out exact commit + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.toolchain }} + components: clippy, rustfmt + + - name: Verify formatting + run: cargo fmt --all --check + + - name: Run unit, integration, process, and golden tests + run: cargo test --locked --all-targets + + - name: Reject Clippy warnings + run: cargo clippy --locked --all-targets -- -D warnings + + - name: Reject rustdoc warnings + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --locked --no-deps diff --git a/Cargo.toml b/Cargo.toml index 857e4e6..28c4f87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "a3s-runtime" version = "0.2.0" edition = "2021" +rust-version = "1.85" description = "Provider-neutral execution contract and client for A3S runtimes" license = "MIT" repository = "https://github.com/A3S-Lab/Runtime" @@ -14,7 +15,7 @@ serde_json = "1" sha2 = "0.10" tempfile = "3" thiserror = "2" -tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/README.md b/README.md index 79564e4..d7304c2 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,9 @@ their owning applications. cross-process locks without following symbolic-link state boundaries - **Logs and Exec**: Expose generation-bound log and exec surfaces only when a provider reports the corresponding capability -- **Conformance Suite**: Exercise the common Task and Service lifecycle against - provider-owned disposable resources +- **Capability-Driven Conformance**: Always run complete Base and Recovery + profiles, automatically activate advertised optional profiles, and reject + missing fixtures, incomplete evidence, or provider inventory leaks ## Runtime Model @@ -82,6 +83,8 @@ A unit specification includes: - an optional digest binding caller-owned execution semantics. All wire records use explicit schema identifiers and reject unknown fields. +`RuntimeUnitSpec` v2 makes the ephemeral-storage quota optional; a provider +needs that resource control only when a specification requests the quota. Protocol validation occurs before state reservation or provider work. ### Observations @@ -110,8 +113,14 @@ The `RuntimeClient` contract exposes: Each mutating request carries its own request ID and optional absolute deadline. An exact retry returns or reconstructs the same logical result. Reusing a -request ID with different content fails with `RequestConflict`. A deadline is -checked independently before provider dispatch. +request ID with different content fails with `RequestConflict`. A completed +receipt remains replayable after its original deadline and after later +lifecycle operations; an expired pending request is not redispatched. A +deadline is checked independently before new provider work. On the first exec +reservation, Runtime persists the smaller of `started_at + timeout_ms` and the +optional caller deadline. `RuntimeDriver::exec` receives that effective +absolute deadline in `deadline_at_ms`, and every pending replay receives the +same value, so retrying cannot restart or extend the execution window. ## Capabilities @@ -132,8 +141,12 @@ provider integration: ```text state root/ -├── locks/ # per-unit cross-process locks -└── units/ # atomic JSON records and request receipts +├── locks/ # short per-unit record locks +├── operations/ # full-operation cross-process leases +└── units/ + └── / + ├── record.json # active unit record + └── requests/ # one durable receipt per request ID ``` The store uses a SHA-256 storage key derived from the validated unit ID. Records @@ -166,34 +179,52 @@ ManagedRuntimeClient Provider `apply` must be idempotent for the supplied unit ID and generation. After an ambiguous acknowledgement, a repeated call must discover or converge -the same resource rather than create another one. Provider-specific labels, -SDK handles, container fields, and transport details stay behind the driver. +the same resource rather than create another one. A successful generation +handoff must retire all older provider generations and verify that exactly the +current resource remains; interrupted handoffs finish on exact retry. +Provider-specific labels, SDK handles, container fields, and transport details +stay behind the driver. ## Conformance -Provider repositories should run `verify_runtime_provider` against real, -disposable infrastructure: +Production provider repositories should implement `RuntimeConformanceFixture` +and run `verify_runtime_profiles` against real, disposable infrastructure: ```rust,ignore -use a3s_runtime::{verify_runtime_provider, RuntimeConformanceCase}; - -let case = RuntimeConformanceCase { - task_apply, - task_remove, - service_apply, - service_stop, - service_remove, -}; - -let report = verify_runtime_provider(client.as_ref(), &case).await?; -assert!(report.task.converges(&task_spec)); -assert!(report.service.converges(&service_spec)); +use a3s_runtime::{verify_runtime_profiles, RuntimeConformanceFixture}; + +let fixture: &dyn RuntimeConformanceFixture = provider_fixture; +let report = verify_runtime_profiles(client.as_ref(), fixture).await?; +assert_eq!(report.inventory_before, report.inventory_after); ``` -The shared suite validates capability matching, exact apply replay, inspection, -stop replay, removal replay, and generation-aware absence for both lifecycle -classes. Provider repositories remain responsible for crash injection, -reconstruction, provider-specific security, and resource-leak tests. +The mandatory Base profile covers successful, failed, and timed-out Tasks; +Service apply, inspect, stop, and removal; exact replay; generation conflicts; +and tombstones. Recovery is mandatory for every production provider. +Networking, Mounts, Health, Resources, Logs, Exec, Security, Outputs, and +Evidence activate from reported capabilities. A fixture must return the shared +stable case IDs and capability claims for every activated profile, perform +cleanup even after a failed profile, and prove that its canonical provider +inventory returned to the pre-run baseline. + +Profile requirements expand to one case ID per advertised behavior rather than +accepting a generic family-level claim. For example, every reported network +mode, mount kind, health probe, and resource control activates its own +configuration and behavioral cases. `NetworkMode::Service` activates both TCP +and UDP because the current protocol has no narrower transport-protocol +capability. Logs separately require filtering, total order, cursor resume, +same-timestamp handling, limits, explicit rotation gaps, terminal retention, +and bounded large records. + +`verify_runtime_provider` remains available as the lower-level successful Task +and Service lifecycle check. It is not, by itself, production certification. +Provider-specific fixtures still own real daemon restart, external deletion, +security, resource-behavior, and destructive cleanup mechanics; the shared +harness owns activation, required case/claim coverage, Base semantics, and the +zero-inventory-delta oracle. + +See the [deep test plan](docs/deep-test-plan.md) for the full contract, +durability, real-provider, fault, performance, soak, and A3S OS release gates. ## Architecture @@ -213,8 +244,11 @@ managed durability and validation provider driver and external runtime ``` -See [ADR 0001](docs/adr/0001-general-runtime-contract.md) for the ownership, -identity, retry, and migration decisions behind the contract. +See [ADR 0001](docs/adr/0001-general-runtime-contract.md) for the general +ownership model, [ADR 0002](docs/adr/0002-complete-protocol-and-operation-semantics.md) +for the completed protocol and operation semantics, and the +[implementation plan](docs/implementation-plan.md) for the dependency-ordered +delivery tasks. ## Development diff --git a/docs/adr/0002-complete-protocol-and-operation-semantics.md b/docs/adr/0002-complete-protocol-and-operation-semantics.md new file mode 100644 index 0000000..424f239 --- /dev/null +++ b/docs/adr/0002-complete-protocol-and-operation-semantics.md @@ -0,0 +1,257 @@ +# ADR 0002: Complete Protocol and Operation Semantics + +- Status: Accepted +- Date: 2026-07-17 +- Decision owners: A3S Runtime maintainers + +## Context + +ADR 0001 established the general Task and Service model, immutable generations, +durable request identity, and provider-neutral ownership boundary. Source and +test-plan review found several places where the public API does not yet provide +enough information to prove those guarantees: + +- some top-level wire types have no schema identifier; +- capability provider identity uses a different grammar from `ProviderId` and + is not checked against the selected driver; +- a newer generation can overwrite durable identity before the old provider + resource has been reconciled; +- file locking protects individual writes but not the asynchronous provider + operation between reservation and completion; +- Task outputs have no matching capability and no reported byte size; +- exec has a request ID but no durable replay receipt; +- request receipts are embedded in one record with a hard 10,000-entry limit; +- deadlines are checked before reservation but do not bound lock waiting and + provider dispatch; +- log, exec, stop-after-loss, and operation-result state semantics are not + explicit enough to form complete test oracles. + +These are pre-1.0 breaking changes. They are resolved together so provider and +consumer integrations migrate once to one coherent contract. + +## Decision + +### 1. Every top-level wire type is explicitly versioned + +The following top-level values carry a `schema` field and reject unknown +fields: + +- capabilities; +- unit specifications and apply/action requests; +- observations, inspections, and removals; +- log queries and log chunks; +- exec requests and exec results; +- durable unit records and request receipts. + +Nested value objects remain versioned by their enclosing top-level schema. +Making `ephemeral_storage_bytes` optional changes unit specifications to +`a3s.runtime.unit-spec.v2`; v1 required a numeric quota. An omitted quota is +encoded explicitly as `null` in v2 and requires no ephemeral-storage +capability. Adding `size_bytes` to output artifacts changes observations to +`a3s.runtime.observation.v2`. Adding typed provider identity and output +capability changes capabilities to `a3s.runtime.capabilities.v3`. The durable +request-journal layout uses `a3s.runtime.unit-record.v2`. Persisting the +effective request deadline changes request receipts to +`a3s.runtime.request-receipt.v2`. + +The Runtime core does not silently reinterpret legacy records or receipts. A +caller that needs old state owns an explicit archival decoder or migration +before starting the new client. + +### 2. Provider identity is one typed value + +`RuntimeCapabilities.provider_id` uses `ProviderId`, serialized as its existing +lowercase string representation. `RuntimeDriver` reports its stable +`ProviderId`, and `ManagedRuntimeClient` rejects capabilities whose ID differs +from the selected driver. + +`RuntimeProviderFactory` construction becomes asynchronous. The registry +validates the created client's capabilities and verifies that the reported ID +matches the registered factory before returning the client. Observations do +not repeat the provider ID: the validated selected client is the authority, +while `provider_resource_id` remains provider-scoped identity. + +### 3. One unit ID has one converged provider generation + +Applying generation N durably makes N the desired generation before provider +dispatch. A driver must reconcile provider resources so that, when apply +returns successfully, exactly one resource generation for that unit remains. +Older generations are retired idempotently by the driver. + +A transient overlap is permitted while the provider performs a handoff, but a +crash and exact retry must converge it to one resource. A caller that needs two +generations alive simultaneously, such as a rolling deployment, uses distinct +unit IDs and owns traffic switching above Runtime. + +This rule lets a provider discover stale resources from provider labels or +equivalent metadata even though the core has already persisted the new desired +record. + +### 4. A state store supplies a cross-process operation lease + +`RuntimeStateStore` exposes an owned, asynchronous, per-unit operation lease. +`ManagedRuntimeClient` holds it across reservation, provider dispatch, and +durable completion for apply, inspect, stop, remove, logs, and exec. + +The file store implements the lease with a separate owner-only advisory-lock +file. Record writes continue to use a shorter record lock, so completion does +not recursively acquire the operation lock. A process crash or cancelled +future releases the lease through operating-system file-handle cleanup. + +Same-unit operations are serialized across tasks and processes. Different unit +IDs remain parallel. Distributed stores must implement an equivalent fenced +lease; a no-op implementation is not conformant. + +### 5. Recovery identity changes only after confirmed loss + +An ambiguous acknowledgement leaves a pending receipt. Retrying it must +discover the same provider resource and may not substitute identity. + +If inspect proves that a previously observed provider resource is absent, the +core durably records `unknown`. A later same-generation apply may adopt one new +provider resource ID. Once that observation is completed, exact replay returns +the replacement without another provider create. + +`unknown` is a confirmed-loss state, not a second `accepted` state. Recovery +may move from `unknown` to a valid provider-backed intermediate or operation +result and may adopt a replacement provider identity, but it cannot regress to +`accepted`. + +### 6. Operation postconditions are explicit + +Apply returns a provider-backed observation that has advanced beyond +`accepted`: + +- a Task returns `succeeded` or `failed`; +- a Service returns `running`, `stopped`, `failed`, or `unknown`; +- `preparing`, `starting`, and `stopping` are inspectable intermediate states, + not successful apply results. + +A Service may be `running` but unhealthy. That is a truthful result and does +not satisfy `converges`. + +Stop returns `stopped`, an already terminal observation, or durable `unknown` +when the provider resource is confirmed lost. It may not report a still-active +state as a successful stop result. Stop never recreates an unknown resource. + +The shared conformance successful fixtures must converge; negative fixtures +also verify failed and unhealthy results. + +### 7. Task outputs are capability-gated and exact + +`RuntimeFeature::OutputArtifacts` advertises Task output collection. A Task +with nonempty `outputs` is rejected before reservation unless the feature is +present. + +`RuntimeOutputArtifact` includes `size_bytes`. A succeeded Task that requested +outputs must report exactly the requested names. Every artifact media type must +match its output specification, every artifact is digest-bound, and +`size_bytes` must not exceed `max_bytes`. A provider without output collection, +including the current Docker driver, does not advertise the feature. + +`IsolationLevel::Confidential` additionally requires the attestation feature. +Usage remains optional observation data unless a future specification requests +it explicitly. + +### 8. Logs and exec have different state policies + +Logs are readable for the current, non-removed generation in any lifecycle +state, including terminal and unknown. This permits postmortem logs. The +provider may return `NotFound` when retained provider logs no longer exist. +Removal closes the Runtime log surface for that unit generation. + +Exec requires the current, non-removed generation to be `running`. It is a +mutating request and therefore participates in durable request replay: + +- reserve an exec receipt before dispatch; +- an exact completed retry returns the stored result without re-execution; +- an ambiguous failure leaves the receipt pending; +- retrying a pending exec uses the same request ID, and an Exec-capable driver + must deduplicate or reattach that request; +- conflicting reuse of the request ID fails before provider dispatch. + +Exec results are stored in the request journal rather than embedded in the +unit record because bounded output may still be large. + +### 9. Request receipts use a durable per-request journal + +The v2 file layout separates the active unit record from receipts: + +```text +state root/ +├── locks/ +├── operations/ +└── units/ + └── / + ├── record.json + └── requests/ + └── .json +``` + +Pending and completed receipts are atomically written and owner-only. Exact +replay is guaranteed until an explicitly removed unit is purged through a +future administrative retention operation. Normal lifecycle methods never +silently discard receipts, and there is no 10,000-operation availability +cliff. + +The public unit record does not need to load every historical result to perform +an operation. Reservations return the one relevant receipt. Tests can enumerate +the journal through a test-only or administrative inspection surface. + +### 10. Deadlines cover queueing and provider work + +Managed operations validate the deadline before capability work, again after +acquiring the operation lease, and use the remaining duration to bound the +provider future. A timeout returns `DeadlineExceeded` and leaves a dispatched +mutating request pending because provider acknowledgement is ambiguous. + +An exact replay whose receipt is already `completed` returns that durable +result before capability or deadline checks, including after the original +absolute deadline and after later lifecycle operations. The core reacquires +the unit lease and lets the state store reconcile a receipt-first crash before +returning. Deadlines constrain unfinished work; they do not invalidate an +already committed response. A pending replay remains subject to its original +deadline and is never redispatched after that deadline expires. + +The request receipt stores the effective absolute deadline captured on first +reservation. For Exec this is the smaller of the first attempt's relative +timeout and optional absolute deadline; a retry cannot restart that relative +timeout window. Before provider dispatch, `ManagedRuntimeClient` replaces the +driver-bound exec request's optional caller deadline with that persisted +effective absolute deadline. The driver therefore receives the same non-null +`deadline_at_ms` on the first dispatch and every pending replay. + +Drivers may enforce a shorter provider-specific timeout. They must never extend +the caller deadline. Exec uses the smaller of its relative `timeout_ms` and an +optional absolute request deadline. + +Logs and inspect remain read operations without a request deadline in this +version; provider adapters must still have a configured transport timeout. + +## Conformance consequences + +The shared conformance suite is split into Base, Recovery, Networking, Mounts, +Health, Resources, Logs, Exec, Security, Outputs, and Evidence profiles. + +- Base and Recovery are mandatory for every production provider. +- An advertised optional capability activates its corresponding profile. +- A provider job fails if a required fixture or provider prerequisite is + absent; it does not silently pass by returning early. +- Generation advancement, operation cancellation, and all mutating crash + windows include provider inventory checks. +- Successful cleanup must return provider and state inventory to the declared + baseline. + +## Consequences + +- The public protocol and durable state schema break once before 1.0. +- Provider adapters must add typed identity, generation reconciliation, + operation timeout handling, and idempotent exec when advertised. +- State-store implementations gain a cross-process lease and separate receipt + journal. +- Exact replay no longer has an arbitrary embedded-record limit. +- Docker truthfully rejects requested outputs and exec until those capabilities + are implemented. +- A3S Box can implement `IsolationLevel::Sandbox` behind the same contract + without adding Box-specific fields. +- Tests can now derive a deterministic oracle for every public operation. diff --git a/docs/deep-test-plan.md b/docs/deep-test-plan.md new file mode 100644 index 0000000..da0d572 --- /dev/null +++ b/docs/deep-test-plan.md @@ -0,0 +1,625 @@ +# A3S Runtime Deep Test Plan + +## 1. Purpose + +This document defines the verification architecture for `A3S-Lab/Runtime`. +The target is not merely a green crate test suite. The target is evidence that +the provider-neutral contract, durable lifecycle coordinator, state store, real +providers, and consuming control paths preserve the same semantics through +retries, crashes, contention, scale, and provider loss. + +All build and runtime execution described here must run on Linux CI runners or +an A3S OS test runner. Developer laptops are not part of the execution matrix. +Servers must obtain source through Git at an exact commit; source archives must +not be copied from a workstation. + +## 2. Scope + +The plan covers five boundaries: + +1. The public contract in `src/contract/`. +2. `ManagedRuntimeClient`, `FileRuntimeStateStore`, provider registration, and + the exported conformance suite. +3. Every concrete `RuntimeDriver` against its real provider. +4. The A3S Cloud projection, node command journal, reconciliation, and + observation path that consume this crate. +5. Performance, fault recovery, security, resource cleanup, and production + canary evidence. + +The A3S OS product runtime APIs that happen to use the word "runtime" are not +automatically in scope. They enter this plan only when they construct or carry +the Rust `a3s-runtime` protocol. + +## 3. Evidence-Based Baseline + +Source inspection on 2026-07-17 established this baseline. It must be refreshed +at the start of an implementation or release campaign. + +| Area | Current evidence | Gap | +| --- | --- | --- | +| Runtime repository CI | No repository workflow is present | A green commit has no independent build or test evidence | +| Core integration tests | `tests/general_runtime.rs` contains 16 lifecycle, conflict, recovery, state, registry, and conformance scenarios | Most boundary combinations, crash points, cross-process races, and capacity limits are untested | +| Shared conformance | One happy-path Task and Service flow with replay checks | Optional features and negative/fault profiles are not covered | +| Real provider | A3S Cloud contains `DockerRuntimeDriver` | No other concrete Runtime driver is implemented | +| Real Docker tests | Three tests cover common conformance, create-before-state-update recovery, and external provider loss | They return success without running unless `A3S_CLOUD_TEST_DOCKER=1` | +| A3S Box | Documented as a future provider | No `RuntimeDriver` exists, so Box conformance cannot currently be claimed | +| Non-functional testing | No benchmark, fuzz, mutation, multi-process crash, or soak harness is present | No regression or durability evidence exists | + +Existing tests are useful regression assets, but their presence is not proof +that they passed for a given commit. Every report must bind results to an exact +Runtime commit, provider commit, provider build, fixture digest, and host. + +## 4. Required Invariants + +These invariants are the basis of every test oracle. + +### 4.1 Protocol and identity + +- Every wire record accepts only its declared schema and fields. +- `(unit_id, generation, canonical_spec_digest)` is immutable. +- A request ID identifies exactly one request kind and digest. +- Exact request replay returns the same logical result. +- Stale generations and conflicting content fail before provider mutation. +- A mutable artifact tag never replaces the declared digest. +- Capability rejection occurs before state reservation and provider dispatch. +- Provider observations, exec results, removals, evidence, and attestations + cannot substitute caller-owned identity. + +### 4.2 Lifecycle and recovery + +- Task convergence means `succeeded`; Service convergence means `running` and, + when configured, `healthy`. +- Terminal observations are immutable. +- Explicit removal creates a durable generation-aware tombstone. +- A missing previously observed provider resource becomes `unknown`, not + `not_found` or success. +- An ambiguous acknowledgement retries the same provider identity and must not + create a duplicate resource. +- After confirmed provider loss has been persisted as `unknown`, a same- + generation apply may adopt one replacement provider identity. Subsequent + replay must return that replacement without another create. +- A deadline at or before the current clock prevents reservation and dispatch. +- Provider operation timeouts remain bounded independently of the pre-dispatch + request deadline check. + +### 4.3 Durability and concurrency + +- A completed state write survives process restart and host reboot. +- A failed or interrupted write never destroys the last valid record. +- A pending receipt survives an ambiguous provider result. +- Concurrent operations preserve every accepted receipt and never create an + untracked provider resource. +- Same-unit serialization and different-unit parallelism have explicit, + measured behavior. +- State paths, files, locks, and temporary writes never follow a symbolic-link + boundary and retain owner-only permissions. +- Corrupt, truncated, mismatched, or future-schema state fails closed before + provider work. + +### 4.4 Provider truthfulness + +- Every advertised capability has a real passing test. +- Every unadvertised optional feature is rejected before provider dispatch. +- Resource limits are verified by provider inspection and observable behavior, + not only by request construction. +- A provider restart, agent restart, or external resource deletion converges to + one durable outcome with zero duplicate resources. +- A successful suite leaves no provider resources, state roots, ports, mounts, + processes, or test volumes outside its declared retention policy. + +## 5. Contract Decisions Required Before P0 Closes + +Tests must not encode accidental behavior. The maintainers must resolve and +record the following decisions in an ADR or contract documentation before the +corresponding release gate is enabled. + +| Decision | Current risk | Required test oracle | +| --- | --- | --- | +| Wire schema boundary | Top-level log, exec, and inspection types do not all carry schema identifiers although the README says all wire records do | Define which types cross a versioned boundary; require a schema on each top-level wire record or narrow the compatibility claim | +| Provider identity binding | Capabilities carry a string validated differently from `ProviderId`, and the managed client does not bind it to a selected factory | Use one grammar and prove reported, registered, and observed provider identities cannot disagree | +| Generation handoff | Reserving a newer generation replaces the stored observation before the old provider resource is necessarily stopped or removed | Define caller, managed client, or driver ownership of the prior resource; prove no orphan on success, error, or crash | +| Cross-operation concurrency | State locking does not span an asynchronous provider call | Define ordering for apply/apply, apply/stop, apply/remove, stop/remove, and generation races | +| Recovery identity | Recovery from `unknown` can differ from ambiguous-ack reattachment | Permit identity replacement only after durable `unknown`; all other identity changes fail | +| Operation postconditions | Apply rejects only `accepted`, while stop can accept any otherwise valid transition | Define whether calls return only converged results or may return transitional observations, then test each allowed result | +| Task output fulfillment | A Task may request outputs, but capabilities do not express output collection and the current Docker driver returns none | Make output support mandatory or capability-gated; prove every requested output is collected, bounded, and digest-bound before convergence | +| Logs by state | Current-generation logs may be useful after Task completion or Service stop | Define allowed states, removal behavior, cursor retention, and provider-loss behavior | +| Exec by state | Generation matching alone does not prove a runnable unit | Require `running`, or explicitly delegate a narrower rule to providers | +| Stop after loss | A provider may be absent while durable state is `unknown` | Define whether stop returns `unknown`, is idempotent success, or requires recovery | +| Receipt retention | Records reject more than 10,000 receipts but have no retention/compaction policy | Define bounded retention without breaking exact replay guarantees | +| Deadline semantics | Deadlines are checked only before dispatch | Define whether drivers receive remaining budget and how late results are persisted | + +## 6. Test Architecture + +The suite is divided into layers so fast deterministic checks run frequently +while destructive checks run only in isolated environments. + +| Layer | Purpose | Environment | Trigger | Maximum target duration | +| --- | --- | --- | --- | --- | +| L0 | Format, lint, docs, unit, golden protocol, property checks | Ephemeral Linux CI | Every pull request | 10 minutes | +| L1 | Managed lifecycle, file state, multi-process contention, deterministic fault driver | Ephemeral Linux CI | Every pull request | 20 minutes | +| L2 | Real provider capability and conformance profiles | Dedicated Linux provider runner | Merge and nightly | 30 minutes per provider | +| L3 | Crash, reboot, disk fault, scale, resource enforcement, and leak tests | Disposable A3S OS worker or VM | Nightly or weekly | 2 hours | +| L4 | Long soak and safe production canary | Dedicated soak worker; shared production only for non-destructive canary | Release candidate | 24 to 72 hours | + +No layer may silently convert an unavailable prerequisite into a passing test. +A skipped provider job must be reported as `SKIPPED`, and a release gate that +requires that provider must fail. + +The initial automation uses the crate's documented commands from the Runtime +repository root: + +```text +cargo fmt --all --check +cargo test --all-targets +cargo clippy --all-targets -- -D warnings +RUSTDOCFLAGS="-D warnings" cargo doc --no-deps +``` + +The real Docker gate runs from the A3S Cloud repository root: + +```text +A3S_CLOUD_TEST_DOCKER=1 cargo test -p a3s-cloud-node-agent \ + --test docker_conformance real_docker_passes_all_advertised_runtime_profiles \ + -- --ignored --exact --nocapture --test-threads=1 +``` + +The Docker test is explicitly ignored in ordinary workspace runs. Its +dedicated job must select the ignored test by exact name and fail unless the +Docker enable flag, isolated provider restart target, and provider socket are +present. This prevents an unavailable provider from being counted as a passing +certification. + +The Runtime crate must declare a minimum supported Rust version before release; +L0 tests that version and current stable on the production Linux architectures. + +## 7. Functional Test Matrix + +Each implemented case receives a stable ID in the test name and evidence +manifest. Boundary tests use `minimum-1`, `minimum`, `maximum`, and `maximum+1` +where the type permits those values. + +### 7.1 Serialization, validation, and digests + +| IDs | Required coverage | +| --- | --- | +| `CT-SCHEMA-*` | Correct, missing, old, future, malformed, and unknown-field behavior for every public record | +| `CT-ID-*` | Empty, 1 byte, maximum length, overlength, control characters, Unicode byte length, path-like characters, and provider ID grammar | +| `CT-DIGEST-*` | Digest algorithm, hexadecimal length, invalid bytes, deterministic output across processes and architectures, and one-field mutation sensitivity | +| `CT-JSON-*` | Golden JSON decode/encode, enum tags, field names, missing required fields, duplicate JSON keys, and map-order independence | +| `CT-URI-*` | Artifact URI schemes, authority/path boundaries, digest binding, credentials, query/fragment, and mutable tags in each provider adapter | +| `CT-PATH-*` | Absolute paths, repeated separators, dot segments, `..`, NUL, CR/LF, maximum length, and mount/secret/output collision policy | + +Golden fixtures are versioned test data. Updating one requires an explicit +schema review; snapshot acceptance is never automatic. + +### 7.2 Unit specifications + +| IDs | Required coverage | +| --- | --- | +| `SPEC-PROCESS-*` | Entrypoint fallback, command/argument counts and sizes, working directory, environment names/values, and deterministic map order | +| `SPEC-MOUNT-*` | Artifact, volume, and tmpfs sources; read-only behavior; zero/overflow size; duplicate names and targets; cross-kind collisions | +| `SPEC-SECRET-*` | Opaque references, environment/file targets, file modes, duplicate names/targets, collision with process environment and mounts, and non-disclosure | +| `SPEC-NET-*` | None/outbound/service modes, TCP/UDP, zero port, duplicate name/socket, 64-port boundary, and service-only publication | +| `SPEC-HEALTH-*` | HTTP/TCP/command probes, named-port binding, status ranges, path validation, timing relationships, start period, and thresholds | +| `SPEC-RESOURCE-*` | CPU, memory, PIDs, optional ephemeral storage, Task timeout, zero values, numeric maxima, and provider conversion overflow | +| `SPEC-ISOLATION-*` | Process, container, sandbox, and confidential requirements; capability rejection; provider mapping and evidence | +| `SPEC-RESTART-*` | Never, always, and on-failure policies; retry boundaries; Task/Service restrictions; provider enforcement | +| `SPEC-CLASS-*` | Every Task/Service restriction for health, timeout, restart policy, and outputs | +| `SPEC-OUTPUT-*` | Names, absolute paths, media types, size limits, duplicates, and succeeded-Task-only observations | +| `SPEC-SEMANTICS-*` | Optional profile digest validation and evidence binding | + +Ephemeral storage is conditional: a provider needs +`ResourceControl::EphemeralStorage` only when the specification requests a +quota. CPU, memory, and PIDs remain required for every unit. + +### 7.3 Capabilities and registry + +Test every required family, duplicate entry, malformed provider identity, +empty optional family, and complete missing-capability ordering. Generate a +spec that independently requires each enum value and optional feature. Verify +that the registry rejects duplicate providers, has no default, never falls +back, and surfaces factory construction failures unchanged. + +Capability claims are tested twice: + +1. Pure matching proves that the contract calculates requirements correctly. +2. Real-provider probes prove that each advertised claim works. + +### 7.4 Observations and transitions + +Generate and verify the full state-transition matrix for Task and Service. +Cover provider identity, build identity, monotonic observation time, start and +finish ordering, terminal timestamps, health, outputs, usage, evidence, +attestation, failure details, and convergence. + +The matrix must include: + +- every allowed transition; +- every forbidden transition; +- same-state refresh; +- all transitions to and from `unknown` allowed by the final contract; +- terminal equality replay and every attempted terminal mutation; +- provider identity substitution before and after confirmed loss; +- a Service attempting `succeeded` and a Task carrying health; +- duplicate output artifacts and evidence bound to another specification. + +### 7.5 Managed operations + +| IDs | Required scenarios | +| --- | --- | +| `LC-APPLY-*` | First apply; pending replay; completed replay; same generation/new request; generation conflict; stale and next generation; rejected capability; expired deadline | +| `LC-INSPECT-*` | Never seen; active refresh; terminal cache; removed tombstone; provider absent to durable `unknown`; malformed provider response | +| `LC-STOP-*` | Running, already stopped, terminal Task, absent, removed, unknown, exact replay, request conflict, unsupported feature, and deadline | +| `LC-REMOVE-*` | Running, stopped, Task terminal, already absent with new request, exact replay, identity substitution, and provider error | +| `LC-LOG-*` | Current/stale/future/removed generation, stdout/stderr filter, limits, strict sequence, cursor resume, invalid cursor, rotation gap, large chunk, and capability rejection | +| `LC-EXEC-*` | Allowed/forbidden states, timeout, exit range, stdout/stderr bounds, truncation, identity substitution, generation checks, replay policy, and capability rejection | + +Every mutation test records state immediately before reservation, after +reservation, after provider completion, and after durable completion. + +## 8. Durable State and Race Matrix + +### 8.1 Filesystem safety + +Run on a real Linux filesystem with hostile fixtures for the root, `locks`, +`units`, lock file, record file, and temporary-file boundary. Cover symbolic +links, hard links where relevant, non-regular files, permission changes, +different umasks, wrong ownership, read-only filesystem, `ENOSPC`, inode +exhaustion, truncated JSON, invalid UTF-8, unknown schema, key mismatch, and a +record exceeding the receipt limit. + +After every successful write, assert directory mode `0700`, file and lock mode +`0600`, valid JSON, matching storage key, and a directory sync. After every +injected failure, assert that either the prior record or the complete new record +is readable; a partially published record is never acceptable. + +### 8.2 Crash points + +Use subprocess tests and named failpoints around: + +1. lock acquisition; +2. initial reservation construction; +3. temporary-file creation; +4. partial and complete writes; +5. file sync; +6. permission tightening; +7. atomic publish; +8. directory sync; +9. provider create/start/stop/remove; +10. observation or removal completion. + +For each point, kill the process, create a new client from the same state root, +replay the same request, and compare provider inventory with durable state. +Cancel the caller future at each asynchronous boundary as a separate case; +dropping a future must not make a later state mutation or provider result +unrecoverable. + +### 8.3 Concurrency cases + +Run each race in thread, task, and independent-process variants where the +boundary permits it: + +- 2, 32, and 128 identical apply requests; +- same specification with distinct request IDs; +- conflicting content with one request ID; +- generations N and N+1 concurrently; +- apply versus inspect, stop, and remove; +- stop versus remove; +- observation refresh versus remove; +- 1,000 different units concurrently; +- 10,001 sequential receipts for one unit. + +Pass criteria are deterministic accepted/error classes, no lost completed +receipt, no invalid record, no deadlock, and exactly the provider resources +allowed by the generation-handoff decision. + +## 9. Provider Conformance Profiles + +Replace the single all-or-nothing conformance path with composable profiles. +The shared Runtime repository owns the oracles; provider repositories own +fixtures and destructive cleanup. + +| Profile | Mandatory evidence | +| --- | --- | +| Base | Valid capabilities, Task success/failure/timeout, Service start/inspect/stop/remove, exact replay, generation conflict, tombstone | +| Recovery | Create-before-ack crash, client restart, provider restart, external deletion to `unknown`, one same-generation replacement, duplicate-resource detection | +| Networking | Every advertised network mode and protocol, loopback publication, outbound denial/allowance, port collision behavior | +| Mounts | Every advertised mount kind, read-only enforcement, persistence, isolation, cleanup | +| Health | Every advertised probe kind, threshold transitions, timeout, start period, unhealthy exit | +| Resources | Every advertised control verified by provider configuration and workload behavior | +| Logs | Stream filtering, total order, cursor resume, same-timestamp records, limit, rotation gap, retention, large records | +| Exec | State policy, timeout, exit code, output bounds, truncation, identity and generation binding | +| Security | Digest pinning, label/metadata tamper, namespace separation, secret handling, least privilege, hostile input | +| Evidence | Usage, evidence claims, profile binding, attestation validity for each advertised optional feature | + +The harness must always run the Base and Recovery profiles. It discovers +optional profiles from capabilities and fails if an advertised capability has +no fixture or passing probe. + +## 10. Current Provider Matrix + +This table describes source claims, not release certification. + +| Capability | Docker driver in A3S Cloud | A3S Box driver | +| --- | --- | --- | +| Task and Service | Advertised | Not implemented | +| OCI/Docker manifest | Advertised with digest-pinned URI | Not implemented at this boundary | +| Isolation | `container` | Intended `sandbox`; no adapter evidence | +| Network | none, outbound, service | Not implemented at this boundary | +| Mounts | named volume, tmpfs | Not implemented at this boundary | +| Health | HTTP, TCP, command | Not implemented at this boundary | +| Resources | CPU, memory, PIDs, Task timeout | Not implemented at this boundary | +| Features | durable identity, stop, remove, logs | Not implemented at this boundary | +| Not advertised | exec, usage, attestation, secrets, artifact mounts, ephemeral quota | No claims are testable yet | + +An A3S Box row may become a release gate only after a real `RuntimeDriver` +exists and maps `IsolationLevel::Sandbox` without provider-specific fields in +the public contract. + +### 10.1 Docker-specific mandatory cases + +- Pull and reuse a digest-pinned image; reject tag-only, digest mismatch, + credentials in the URI, malformed registry responses, and interrupted pull. +- Validate all managed labels and reject label tampering, wrong node, + namespace collision, and multiple matching containers. +- Prove Task exit 0, nonzero exit, signal exit, timeout, and daemon error. +- Prove HTTP, TCP, and command health success and failure thresholds. +- Verify CPU, memory/swap, PIDs, network mode, loopback port binding, restart + policy, volume mode, and hardened tmpfs through Docker inspect and behavior. +- Exercise stdout/stderr ordering, nanosecond timestamp ties, cursor replay, + log rotation gap, filters, and one-MiB record rejection. +- Restart the node agent and Docker daemon at each recovery window. +- Count labeled containers and test volumes before and after every case; both + deltas must be zero after cleanup. + +The Docker job must set `A3S_CLOUD_TEST_DOCKER=1` explicitly and verify the +daemon before the test binary starts. A missing socket is a failed provider job, +not a passed test with early returns. + +## 11. Consumer End-to-End Matrix + +The first real consumer is A3S Cloud. Its release suite must prove: + +- an immutable `WorkloadRevision` projects to the expected Service spec and + digest; +- scheduler capability matching rejects an ineligible node before dispatch; +- the node command journal deduplicates exact commands and rejects conflicts; +- command lease expiry, redelivery, observation loss, and reordering converge; +- control-plane and node-agent restart preserve one provider resource; +- generation updates and rollback follow the generation-handoff decision; +- runtime observations remain bound to the selected node and deployment; +- ordered log cursors survive transport batching and reconnect; +- stop, cancel, deferred cleanup, and reconciliation leave no untracked unit. + +Bench or another caller must add a separate profile-projection suite before it +can claim Runtime compatibility. Product-specific scoring, privacy, and +scheduling remain outside the Runtime crate, but their projection digest and +generic resource/isolation requirements are in scope. + +## 12. Fault Injection Matrix + +| Fault | Expected result | +| --- | --- | +| Transport error before provider mutation | Pending receipt; exact retry dispatches safely | +| Provider create succeeds, acknowledgement is lost | Retry reattaches the same resource; count remains one | +| Provider resource is externally deleted | Inspect persists `unknown`; one reapply creates one replacement | +| Agent is killed at every state/provider boundary | Restart and replay converge without invalid state or orphan | +| Provider daemon restarts | Operations remain bounded and later reconcile without duplicates | +| Host reboots after file or provider sync | Durable state and provider inventory converge | +| Disk becomes full or read-only | Prior record remains valid; no provider work after failed reservation | +| State file is truncated or tampered | Fail closed; no provider mutation | +| Clock jumps backward/forward | Request deadline and observation monotonicity remain deterministic | +| Duplicate provider resources are injected | Inspect/apply fails closed and reports the invariant violation | +| Network is partitioned or latency exceeds timeout | Bounded error, durable pending state where acknowledgement is ambiguous | +| Caller future is cancelled or provider task panics | Durable state remains replayable and later work cannot publish an untracked result | + +Daemon restart, host reboot, disk faults, and broad network faults must never be +run on a shared production node. They require a disposable worker or nested VM +in the A3S OS environment. + +## 13. Security and Adversarial Testing + +- Fuzz every public JSON decoder, validation function, digest function, state + record decoder, log cursor decoder, and provider artifact URI adapter. +- Add property tests for validation boundaries and transition invariants. +- Run contract-only tests under Miri where supported and use a concurrency + model checker for any in-memory coordinator introduced later. +- Verify state path confinement against symlink replacement and time-of-check/ + time-of-use attempts from another process. +- Verify provider namespaces cannot discover, stop, remove, log, or exec units + owned by another namespace or node. +- Verify error, audit, evidence, and log artifacts never contain secret values + or registry credentials. +- Verify OCI digest pinning after pull and before create. +- Run dependency license, advisory, and supply-chain policy checks as a release + input; they supplement rather than replace behavioral tests. + +Fuzz regressions are checked in as minimal fixtures. Release candidates require +a bounded continuous fuzz run for every target and zero unresolved crashes, +panics, hangs, or uncontrolled allocations. + +## 14. Performance and Capacity Plan + +### 14.1 Core benchmarks + +Measure minimum, typical, and maximum-size records for: + +- specification validation and digest calculation; +- capability matching; +- observation and transition validation; +- state reserve, load, update, removal, and exact replay; +- records containing 1, 100, 1,000, and 10,000 receipts; +- same-unit contention and different-unit throughput at 1, 8, 32, and 128 + clients. + +### 14.2 Provider benchmarks + +Measure cold-image and warm-image Task apply, Service apply-to-healthy, +inspect, stop, remove, log pagination, provider restart recovery, and external +loss recovery. Record p50, p95, p99, throughput, CPU, RSS, file descriptors, +disk bytes, network bytes, and resource cleanup latency. + +### 14.3 Initial gates + +- Correctness counters allow zero lost receipts, duplicate resources, corrupt + records, unexplained log gaps, or leaked resources. +- A pull request fails when a controlled benchmark shows a statistically + significant regression greater than 10% in median or 15% in p95 against the + same runner class. +- Absolute latency SLOs are frozen only after three clean baselines on the + production-equivalent runner; they must not be guessed from laptop results. +- A 10,000-cycle churn test must finish with zero provider resources, mounts, + ports, state roots outside retention, and a non-growing file descriptor + count. +- A 24-hour merge soak and 72-hour release soak must show no monotonic RSS, + state-temporary-file, or provider-resource growth after workload count + returns to baseline. + +## 15. A3S OS Execution Safety + +### 15.1 Git-only deployment + +The runner checks out exact commits without uploading source: + +```text +git fetch --prune origin +git worktree add --detach /var/tmp/a3s-runtime-tests/ +``` + +Provider and consumer repositories use their own exact SHAs. The evidence +manifest records all of them. A dirty worktree is rejected. + +### 15.2 Isolation + +Every run has: + +- a globally unique run ID and provider namespace; +- a dedicated state root and evidence directory; +- pinned fixture image digests; +- loopback-only dynamic service ports; +- CPU, memory, PIDs, disk, duration, and concurrency limits; +- a process-level cleanup trap and an independent TTL sweeper; +- pre-run and post-run inventories keyed by managed labels. + +Shared production nodes run only L4 canaries: one bounded Task, one loopback +Service, inspect, logs when supported, stop, and remove. They never restart a +daemon, reboot the host, corrupt state, fill disk, alter firewall policy, use +production secrets, or mount production data. + +L3 and destructive L4 work runs on a disposable A3S OS worker that matches the +production kernel, architecture, provider version, filesystem, and security +policy. + +### 15.3 Preflight and stop conditions + +Preflight records host identity, kernel, architecture, cgroup mode, filesystem, +provider build, available CPU/memory/disk/inodes, existing managed resources, +and active deployments. The run does not start without the configured safety +headroom. + +Abort immediately on namespace collision, unexpected non-test resource +selection, failed cleanup, host free-space/inode threshold, sustained host load +threshold, production service error-budget burn, or a provider invariant +violation. Cleanup and evidence capture still run after an abort. + +## 16. Evidence Bundle + +Each run produces one immutable bundle: + +```text +evidence// +├── manifest.json +├── environment.json +├── preflight.json +├── junit.xml +├── coverage/ +├── benchmarks/ +├── faults/ +├── provider-inventory-before.json +├── provider-inventory-after.json +├── logs/ +└── cleanup.json +``` + +`manifest.json` contains run ID, timestamps, trigger, Runtime/provider/consumer +SHAs, fixture digests, selected layers and case IDs, configuration digest, and +final status. `cleanup.json` lists every created resource and proves removal or +an explicitly retained diagnostic artifact. Evidence is redacted before +upload, checksummed, retained by release policy, and linked from the commit or +release check. + +## 17. Delivery Phases + +### P0: Deterministic core gate + +- Resolve the contract decisions in Section 5. +- Add Linux CI for format, lint, docs, unit, integration, and compatibility + fixtures. +- Declare and test the minimum supported Rust version and current stable. +- Add shared fixture builders, stable case IDs, golden JSON, property tests, + and transition-matrix generation. +- Add a deterministic driver with call tracing and failures before and after + every provider boundary. +- Make skipped mandatory prerequisites fail their owning job. + +Exit: L0 and L1 are required checks, all protocol/lifecycle rows have an +identified test, and no unresolved P0 semantic decision remains. + +### P1: Durable recovery gate + +- Add multi-process state contention, failpoint, process-kill, permission, + corrupt-state, `ENOSPC`, and restart tests. +- Implement and verify the chosen generation and concurrent-operation model. +- Define and test receipt retention/compaction. + +Exit: every crash point preserves a valid durable state and produces no +untracked provider mutation in the deterministic provider model. + +### P2: Real Docker certification + +- Expand conformance into the profiles in Section 9. +- Run the mandatory Docker job on a dedicated Linux runner with the gate + explicitly enabled. +- Add Docker fault, resource, log, namespace, and leak cases. +- Run A3S Cloud consumer restart and reconciliation flows. + +Exit: every advertised Docker capability has passing real-provider evidence; +Base and Recovery pass; pre/post resource deltas are zero. + +### P3: Performance, adversarial, and soak gate + +- Add stable benchmark runners, fuzz targets, security cases, 10,000-cycle + churn, and 24/72-hour soak automation. +- Establish three clean baselines and freeze provider-specific SLOs. + +Exit: all correctness, regression, leak, and soak gates in Section 14 pass. + +### P4: Additional provider certification and production canary + +- For each new driver, supply fixtures for every advertised conformance + profile. +- Add A3S Box only after its `RuntimeDriver` exists and verify the `sandbox` + isolation mapping on production-equivalent Linux. +- Run the bounded L4 canary and verify cleanup. + +Exit: the provider matrix is evidence-backed for the release SHAs, and the +production canary bundle proves zero leaked resources and no safety stop. + +## 18. Release Gates + +A Runtime release is blocked unless: + +1. L0 and L1 pass for the exact release commit. +2. Golden compatibility changes have explicit schema approval. +3. The transition, request replay, generation, deadline, and state durability + matrices have no missing P0 row. +4. Base and Recovery pass for every provider declared production-supported. +5. Every advertised optional capability has passing profile evidence. +6. Required consumer end-to-end flows pass at pinned consumer commits. +7. Performance regression, resource leak, fuzz, and soak gates pass at the + release level required by the change risk. +8. Provider inventory after the final run matches the pre-run baseline. +9. The evidence bundle is complete, redacted, checksummed, and reviewable. + +Passing a mock-only suite, an environment-gated test that did not execute, or a +narrow provider smoke test is not sufficient evidence for a release claim. diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md new file mode 100644 index 0000000..28a222a --- /dev/null +++ b/docs/implementation-plan.md @@ -0,0 +1,162 @@ +# A3S Runtime Implementation Plan + +## Goal + +Implement the complete contract accepted by ADR 0001 and ADR 0002, certify all +advertised provider capabilities, and satisfy the release gates in the deep +test plan. Work is ordered by dependency; a later task cannot claim completion +from mocks when its required real-provider evidence is absent. + +## Working rules + +- Preserve user changes and keep each task reviewable. +- Write or update tests before completing the corresponding implementation. +- Run Cargo, provider, fault, performance, and soak validation only on Linux CI + or A3S OS. Do not run those workloads on a developer laptop. +- A3S OS obtains every repository through Git at an exact commit. +- Shared production nodes run only the bounded canary defined by the deep test + plan. Destructive tests require a disposable production-equivalent worker. +- Code and documentation are English, and product configuration remains ACL. + +## Task graph + +| ID | Task | Depends on | Completion evidence | +| --- | --- | --- | --- | +| R00 | Freeze baseline and task manifest | None | Exact Runtime and consumer SHAs, dirty-file ownership, and capability inventory recorded | +| R01 | Version every top-level wire record | R00 | Golden JSON for valid/old/future/unknown-field cases; Cloud wire consumers compile | +| R02 | Unify and bind provider identity | R01 | Typed capability ID, driver/factory checks, mismatch tests, Docker migration | +| R03 | Complete output and confidential capability matching | R01 | Output/attestation rejection before state work; exact output validation tests | +| R04 | Define and enforce operation postconditions | R01 | Full Task/Service result-state matrix; log and exec state tests | +| R05 | Bound deadlines across queue and dispatch | R04 | Deterministic clock tests for pre-lock, post-lock, provider timeout, and pending replay | +| R06 | Add per-unit cross-process operation leases | R04 | Same-unit race serialization and different-unit parallelism in independent processes | +| R07 | Replace embedded receipts with request journal v2 | R06 | Atomic receipt tests, restart replay, 10,001 requests, permissions, corruption handling | +| R08 | Make exec durably idempotent | R07 | Exact replay executes once; conflict, cancellation, timeout, and large-output tests | +| R09 | Enforce generation reconciliation | R06 | N to N+1 success/failure/crash tests with exactly one final provider resource | +| R10 | Build deterministic fault driver and transition generator | R01-R09 | Every state edge and provider boundary has a stable case ID and oracle | +| R11 | Split shared conformance into capability profiles | R10 | Base/Recovery mandatory; optional advertised profiles auto-run and cannot silently skip | +| R12 | Add Runtime repository CI and compatibility gate | R01-R11 | Required Linux MSRV/stable format, lint, docs, unit, integration, and golden checks | +| R13 | Certify Docker identity, lifecycle, and generation handling | R09-R12 | Real Docker Base and Recovery profiles pass; zero duplicate/leaked containers | +| R14 | Certify Docker network, mounts, health, resources, and logs | R11-R13 | Every advertised Docker capability has inspect plus behavioral evidence | +| R15 | Complete A3S Cloud consumer recovery paths | R13-R14 | Projection, journal, restart, redelivery, reconciliation, logs, cancel, and cleanup E2E pass | +| R16 | Implement A3S Box RuntimeDriver | R11 | Typed `sandbox` capability mapping with no provider-specific public fields | +| R17 | Certify A3S Box profiles | R16 | Base/Recovery and every advertised Box profile pass on production-equivalent Linux | +| R18 | Add fuzz, property, and security campaigns | R10-R17 | All declared targets run, regressions checked in, zero unresolved crash/hang/leak | +| R19 | Add benchmarks and regression budgets | R10-R17 | Three clean same-runner baselines and enforced median/p95 budgets | +| R20 | Add churn, fault, and 24/72-hour soak automation | R13-R19 | Evidence bundles prove zero resource/FD/state growth and all stop conditions | +| R21 | Run bounded A3S OS production canary | R12-R20 | Git-only exact-SHA canary passes with complete cleanup and no safety stop | +| R22 | Perform release completion audit | R01-R21 | Every deep-test-plan release gate maps to authoritative passing evidence | + +## Work packages + +### Package A: Protocol completeness (`R00`-`R05`) + +Deliverables: + +- ADR 0002 implementation; +- schema constants and fields for inspection, log, and exec surfaces; +- `ProviderId` as the sole provider identity type; +- output artifact sizes and `OutputArtifacts` capability; +- confidential-attestation capability matching; +- explicit apply, stop, logs, and exec state guards; +- deadline wrappers with deterministic tests; +- golden wire fixtures and schema compatibility tests; +- declared Rust MSRV. + +Exit gate: every public wire value has an explicit version and every operation +has a deterministic state/deadline oracle. + +### Package B: Durable coordination (`R06`-`R10`) + +Deliverables: + +- independent operation and record locks; +- v2 unit directory and request journal; +- durable exec receipts; +- generation reconciliation contract and deterministic provider model; +- process-level crash/failpoint harness; +- transition matrix and race matrix; +- hostile filesystem, permission, corruption, disk-full, and cancellation + cases. + +Exit gate: every mutating crash point can replay after a new process starts, +with valid state and no untracked provider resource. + +### Package C: Shared certification (`R11`-`R12`) + +Deliverables: + +- composable Base, Recovery, Networking, Mounts, Health, Resources, Logs, Exec, + Security, Outputs, and Evidence conformance profiles; +- fixture traits and cleanup inventory contracts; +- CI jobs that fail when a mandatory provider prerequisite is missing; +- MSRV and stable validation, documentation, golden compatibility, and + cross-architecture digest evidence. + +Exit gate: mock and file-state correctness is independently reproducible for an +exact commit on Linux CI. + +### Package D: Docker and A3S Cloud (`R13`-`R15`) + +Deliverables: + +- stale-generation container discovery and cleanup; +- provider identity and label-tamper enforcement; +- Task success/failure/timeout and Service health profiles; +- network, volume, tmpfs, CPU, memory, PIDs, restart, and log-cursor probes; +- node-agent/Docker restart and external-deletion recovery; +- Cloud projection, command journal, redelivery, reconciliation, cancellation, + log transport, and cleanup E2E tests. + +Exit gate: real Docker and Cloud evidence covers every source-advertised +capability, and post-run provider inventory equals its baseline. + +### Package E: A3S Box (`R16`-`R17`) + +Deliverables: + +- a driver hosted at the provider integration boundary; +- digest-pinned OCI apply/inspect/stop/remove/logs/exec mapping; +- stable unit/generation/request labels or metadata; +- `IsolationLevel::Sandbox` mapping; +- output, usage, attestation, secret, network, mount, and resource claims only + when implemented and tested; +- crash, shim loss, VM loss, host restart, and cleanup profiles. + +Exit gate: A3S Box passes Base and Recovery plus every capability it advertises +on production-equivalent A3S OS Linux. + +### Package F: Non-functional release proof (`R18`-`R22`) + +Deliverables: + +- fuzz/property/security campaigns; +- controlled core and provider benchmarks; +- 10,000-cycle churn, resource leak detection, fault automation, and 24/72-hour + soak; +- immutable evidence bundles; +- bounded production canary; +- requirement-by-requirement release audit. + +Exit gate: all nine release gates in the deep test plan have direct, +reviewable, exact-SHA evidence. + +## Immediate execution order + +1. Preserve the existing optional ephemeral-storage and `unknown` recovery + changes as inputs to ADR 0002. +2. Implement `R01`-`R04` with contract tests and update A3S Cloud constructors + in the same compatibility change. +3. Push an exact feature commit so the A3S OS runner can build and test through + Git; do not upload a working tree archive. +4. Implement `R05`-`R10`, rerunning the server-side core matrix after each + durable-state slice. +5. Enable and expand real Docker conformance before starting the A3S Box + adapter, so the shared provider oracles are proven by one real driver first. + +## Completion rule + +The project is complete only when `R22` can map every explicit invariant, +profile, performance gate, soak gate, production-safety condition, and cleanup +requirement to authoritative evidence. An unexecuted environment-gated test, +mock-only result, intended provider feature, or passing narrow smoke test is not +completion evidence. diff --git a/src/conformance.rs b/src/conformance.rs index 400de0d..0d40c37 100644 --- a/src/conformance.rs +++ b/src/conformance.rs @@ -3,6 +3,16 @@ use crate::contract::{ RuntimeRemoval, RuntimeUnitClass, RuntimeUnitState, }; use crate::{RuntimeClient, RuntimeError, RuntimeResult}; +use std::collections::BTreeSet; + +mod profiles; + +pub use profiles::{ + required_runtime_profiles, runtime_profile_requirements, verify_runtime_profiles, + RuntimeConformanceFixture, RuntimeConformanceInventory, RuntimeConformanceProfile, + RuntimeConformanceProfileEvidence, RuntimeConformanceProfileRequirements, + RuntimeConformanceSuiteReport, +}; /// Provider-owned inputs for the destructive Runtime conformance suite. /// @@ -62,6 +72,122 @@ pub struct RuntimeConformanceReport { pub service_removal: RuntimeRemoval, } +/// Complete provider-owned inputs for the mandatory Base profile. +/// +/// Failure and timeout Tasks must deterministically return a `failed` +/// observation. The generation-conflict pair must target the same unit and +/// generation with different canonical specifications. +#[derive(Debug, Clone)] +pub struct RuntimeBaseConformanceCase { + pub lifecycle: RuntimeConformanceCase, + pub task_failure_apply: RuntimeApplyRequest, + pub task_failure_remove: RuntimeActionRequest, + pub task_timeout_apply: RuntimeApplyRequest, + pub task_timeout_remove: RuntimeActionRequest, + pub generation_apply: RuntimeApplyRequest, + pub generation_conflict_apply: RuntimeApplyRequest, + pub generation_remove: RuntimeActionRequest, +} + +impl RuntimeBaseConformanceCase { + pub fn validate(&self) -> Result<(), String> { + self.lifecycle.validate()?; + for (label, apply, remove) in [ + ( + "failure", + &self.task_failure_apply, + &self.task_failure_remove, + ), + ( + "timeout", + &self.task_timeout_apply, + &self.task_timeout_remove, + ), + ] { + apply.validate()?; + remove.validate()?; + if apply.spec.class != RuntimeUnitClass::Task { + return Err(format!("conformance {label} fixture must describe a Task")); + } + validate_action(remove, apply)?; + } + + self.generation_apply.validate()?; + self.generation_conflict_apply.validate()?; + self.generation_remove.validate()?; + if self.generation_apply.spec.class != RuntimeUnitClass::Service { + return Err("conformance generation fixture must describe a Service".into()); + } + if self.generation_apply.spec.unit_id != self.generation_conflict_apply.spec.unit_id + || self.generation_apply.spec.generation + != self.generation_conflict_apply.spec.generation + { + return Err( + "conformance generation-conflict fixtures must target one generation".into(), + ); + } + if self.generation_apply.spec.digest()? == self.generation_conflict_apply.spec.digest()? { + return Err( + "conformance generation-conflict fixtures must have different content".into(), + ); + } + validate_action(&self.generation_remove, &self.generation_apply)?; + + let unit_ids = [ + self.lifecycle.task_apply.spec.unit_id.as_str(), + self.lifecycle.service_apply.spec.unit_id.as_str(), + self.task_failure_apply.spec.unit_id.as_str(), + self.task_timeout_apply.spec.unit_id.as_str(), + self.generation_apply.spec.unit_id.as_str(), + ]; + if unit_ids.iter().copied().collect::>().len() != unit_ids.len() { + return Err("Base conformance fixtures must use distinct unit IDs".into()); + } + + let request_ids = [ + self.lifecycle.task_apply.request_id.as_str(), + self.lifecycle.task_remove.request_id.as_str(), + self.lifecycle.service_apply.request_id.as_str(), + self.lifecycle.service_stop.request_id.as_str(), + self.lifecycle.service_remove.request_id.as_str(), + self.task_failure_apply.request_id.as_str(), + self.task_failure_remove.request_id.as_str(), + self.task_timeout_apply.request_id.as_str(), + self.task_timeout_remove.request_id.as_str(), + self.generation_apply.request_id.as_str(), + self.generation_conflict_apply.request_id.as_str(), + self.generation_remove.request_id.as_str(), + ]; + if request_ids.iter().copied().collect::>().len() != request_ids.len() { + return Err("Base conformance fixtures must use unique request IDs".into()); + } + Ok(()) + } + + pub(crate) fn specifications(&self) -> [&crate::contract::RuntimeUnitSpec; 6] { + [ + &self.lifecycle.task_apply.spec, + &self.lifecycle.service_apply.spec, + &self.task_failure_apply.spec, + &self.task_timeout_apply.spec, + &self.generation_apply.spec, + &self.generation_conflict_apply.spec, + ] + } +} + +/// Evidence returned after the complete mandatory Base profile passes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeBaseConformanceReport { + pub lifecycle: RuntimeConformanceReport, + pub failed_task: RuntimeObservation, + pub failed_task_removal: RuntimeRemoval, + pub timed_out_task: RuntimeObservation, + pub timed_out_task_removal: RuntimeRemoval, + pub generation: RuntimeObservation, + pub generation_removal: RuntimeRemoval, +} + /// Runs the destructive provider-neutral lifecycle conformance suite. /// /// Provider-specific tests remain responsible for crash injection and resource @@ -161,6 +287,129 @@ pub async fn verify_runtime_provider( }) } +/// Runs every mandatory Base-profile oracle, including negative Task results +/// and a same-generation content conflict. +pub async fn verify_runtime_base( + client: &dyn RuntimeClient, + case: &RuntimeBaseConformanceCase, +) -> RuntimeResult { + case.validate().map_err(RuntimeError::InvalidRequest)?; + let capabilities = client.capabilities().await?; + capabilities.validate().map_err(RuntimeError::Protocol)?; + for spec in case.specifications() { + let missing = capabilities + .missing_for(spec) + .map_err(RuntimeError::Protocol)?; + if !missing.is_empty() { + return Err(RuntimeError::UnsupportedCapabilities(missing)); + } + } + + let lifecycle = verify_runtime_provider(client, &case.lifecycle).await?; + let (failed_task, failed_task_removal) = verify_failed_task( + client, + "failed Task", + &case.task_failure_apply, + &case.task_failure_remove, + ) + .await?; + let (timed_out_task, timed_out_task_removal) = verify_failed_task( + client, + "timed-out Task", + &case.task_timeout_apply, + &case.task_timeout_remove, + ) + .await?; + + let generation = client.apply(&case.generation_apply).await?; + generation + .validate_against(&case.generation_apply.spec) + .map_err(RuntimeError::Protocol)?; + if !generation.converges(&case.generation_apply.spec) { + return Err(RuntimeError::Protocol( + "Base generation fixture did not converge".into(), + )); + } + require_equal( + "duplicate generation apply", + &generation, + &client.apply(&case.generation_apply).await?, + )?; + match client.apply(&case.generation_conflict_apply).await { + Err(RuntimeError::GenerationConflict { + unit_id, + generation: rejected_generation, + }) if unit_id == case.generation_apply.spec.unit_id + && rejected_generation == case.generation_apply.spec.generation => {} + Err(error) => return Err(error), + Ok(_) => { + return Err(RuntimeError::Protocol( + "Base generation conflict unexpectedly succeeded".into(), + )); + } + } + let generation_removal = client.remove(&case.generation_remove).await?; + require_equal( + "duplicate generation removal", + &generation_removal, + &client.remove(&case.generation_remove).await?, + )?; + require_absent( + "removed generation fixture inspection", + client.inspect(&generation.unit_id).await?, + generation.generation, + )?; + + Ok(RuntimeBaseConformanceReport { + lifecycle, + failed_task, + failed_task_removal, + timed_out_task, + timed_out_task_removal, + generation, + generation_removal, + }) +} + +async fn verify_failed_task( + client: &dyn RuntimeClient, + label: &str, + apply: &RuntimeApplyRequest, + remove: &RuntimeActionRequest, +) -> RuntimeResult<(RuntimeObservation, RuntimeRemoval)> { + let observation = client.apply(apply).await?; + observation + .validate_against(&apply.spec) + .map_err(RuntimeError::Protocol)?; + if observation.state != RuntimeUnitState::Failed { + return Err(RuntimeError::Protocol(format!( + "conformance {label} did not reach failed" + ))); + } + require_equal( + &format!("duplicate {label} apply"), + &observation, + &client.apply(apply).await?, + )?; + require_equal( + &format!("terminal {label} inspection"), + &observation, + &require_found(label, client.inspect(&observation.unit_id).await?)?, + )?; + let removal = client.remove(remove).await?; + require_equal( + &format!("duplicate {label} removal"), + &removal, + &client.remove(remove).await?, + )?; + require_absent( + &format!("removed {label} inspection"), + client.inspect(&observation.unit_id).await?, + observation.generation, + )?; + Ok((observation, removal)) +} + fn validate_action( action: &RuntimeActionRequest, apply: &RuntimeApplyRequest, @@ -176,7 +425,7 @@ fn validate_action( fn require_found(label: &str, inspection: RuntimeInspection) -> RuntimeResult { match inspection { - RuntimeInspection::Found { observation } => Ok(*observation), + RuntimeInspection::Found { observation, .. } => Ok(*observation), RuntimeInspection::NotFound { .. } => Err(RuntimeError::Protocol(format!( "{label} unexpectedly returned not found" ))), diff --git a/src/conformance/profiles.rs b/src/conformance/profiles.rs new file mode 100644 index 0000000..a85e7fa --- /dev/null +++ b/src/conformance/profiles.rs @@ -0,0 +1,507 @@ +use super::{verify_runtime_base, RuntimeBaseConformanceCase, RuntimeBaseConformanceReport}; +use crate::contract::{ + HealthCheckKind, MountKind, NetworkMode, ResourceControl, RuntimeCapabilities, RuntimeFeature, +}; +use crate::{RuntimeClient, RuntimeError, RuntimeResult}; +use async_trait::async_trait; +use std::collections::{BTreeMap, BTreeSet}; + +/// Composable provider conformance profiles owned by the shared Runtime suite. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum RuntimeConformanceProfile { + Base, + Recovery, + Networking, + Mounts, + Health, + Resources, + Logs, + Exec, + Security, + Outputs, + Evidence, +} + +impl RuntimeConformanceProfile { + pub const fn as_str(self) -> &'static str { + match self { + Self::Base => "base", + Self::Recovery => "recovery", + Self::Networking => "networking", + Self::Mounts => "mounts", + Self::Health => "health", + Self::Resources => "resources", + Self::Logs => "logs", + Self::Exec => "exec", + Self::Security => "security", + Self::Outputs => "outputs", + Self::Evidence => "evidence", + } + } +} + +/// Exact shared case IDs and capability claims required for one profile. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeConformanceProfileRequirements { + pub profile: RuntimeConformanceProfile, + pub case_ids: BTreeSet, + pub capability_claims: BTreeSet, +} + +/// Provider evidence returned for one non-Base profile. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeConformanceProfileEvidence { + pub profile: RuntimeConformanceProfile, + pub case_ids: BTreeSet, + pub capability_claims: BTreeSet, +} + +/// Canonical pre/post provider inventory. Keys can represent provider +/// resources, volumes, ports, mounts, processes, or other retained objects; +/// values are provider-owned stable state digests. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RuntimeConformanceInventory { + pub entries: BTreeMap, +} + +/// Complete report for one profile-driven provider certification run. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeConformanceSuiteReport { + pub base: RuntimeBaseConformanceReport, + pub profiles: Vec, + pub inventory_before: RuntimeConformanceInventory, + pub inventory_after: RuntimeConformanceInventory, +} + +/// Provider integration boundary for destructive, capability-specific +/// fixtures. No method has a success-by-default implementation: a production +/// provider must explicitly declare, execute, clean up, and inventory every +/// activated profile. +#[async_trait] +pub trait RuntimeConformanceFixture: Send + Sync { + fn base_case(&self) -> &RuntimeBaseConformanceCase; + + fn available_profiles(&self) -> BTreeSet; + + async fn inventory(&self) -> RuntimeResult; + + async fn run_profile( + &self, + client: &dyn RuntimeClient, + capabilities: &RuntimeCapabilities, + profile: RuntimeConformanceProfile, + ) -> RuntimeResult; + + async fn cleanup(&self) -> RuntimeResult<()>; +} + +/// Derives mandatory and capability-triggered profiles from source-reported +/// capabilities. Base and Recovery are unconditional. +pub fn required_runtime_profiles( + capabilities: &RuntimeCapabilities, +) -> RuntimeResult> { + capabilities.validate().map_err(RuntimeError::Protocol)?; + let mut profiles = BTreeSet::from([ + RuntimeConformanceProfile::Base, + RuntimeConformanceProfile::Recovery, + ]); + if !capabilities.network_modes.is_empty() { + profiles.insert(RuntimeConformanceProfile::Networking); + } + if !capabilities.mount_kinds.is_empty() { + profiles.insert(RuntimeConformanceProfile::Mounts); + } + if !capabilities.health_check_kinds.is_empty() { + profiles.insert(RuntimeConformanceProfile::Health); + } + if !capabilities.resource_controls.is_empty() { + profiles.insert(RuntimeConformanceProfile::Resources); + } + if capabilities.supports_feature(RuntimeFeature::Logs) { + profiles.insert(RuntimeConformanceProfile::Logs); + } + if capabilities.supports_feature(RuntimeFeature::Exec) { + profiles.insert(RuntimeConformanceProfile::Exec); + } + if !capabilities.isolation_levels.is_empty() + || capabilities.supports_feature(RuntimeFeature::SecretReferences) + { + profiles.insert(RuntimeConformanceProfile::Security); + } + if capabilities.supports_feature(RuntimeFeature::OutputArtifacts) { + profiles.insert(RuntimeConformanceProfile::Outputs); + } + if capabilities.supports_feature(RuntimeFeature::Usage) + || capabilities.supports_feature(RuntimeFeature::Attestation) + { + profiles.insert(RuntimeConformanceProfile::Evidence); + } + Ok(profiles) +} + +/// Returns the stable shared requirements a provider evidence record must +/// cover for the selected profile and capability set. +pub fn runtime_profile_requirements( + capabilities: &RuntimeCapabilities, + profile: RuntimeConformanceProfile, +) -> RuntimeResult { + capabilities.validate().map_err(RuntimeError::Protocol)?; + let mut case_ids = base_case_ids(profile) + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + extend_capability_case_ids(capabilities, profile, &mut case_ids); + if profile == RuntimeConformanceProfile::Security + && capabilities.supports_feature(RuntimeFeature::SecretReferences) + { + case_ids.insert("SECURITY-SECRET-NONDISCLOSURE".into()); + } + if profile == RuntimeConformanceProfile::Evidence + && capabilities.supports_feature(RuntimeFeature::Attestation) + { + case_ids.insert("EVIDENCE-ATTESTATION-VALIDITY".into()); + } + if profile == RuntimeConformanceProfile::Evidence + && capabilities.supports_feature(RuntimeFeature::Usage) + { + case_ids.insert("EVIDENCE-USAGE-VALIDITY".into()); + } + let capability_claims = capability_claims(capabilities, profile); + Ok(RuntimeConformanceProfileRequirements { + profile, + case_ids, + capability_claims, + }) +} + +/// Runs Base internally, activates every required/declared profile, validates +/// typed evidence, always requests provider cleanup, and rejects any inventory +/// delta. +pub async fn verify_runtime_profiles( + client: &dyn RuntimeClient, + fixture: &dyn RuntimeConformanceFixture, +) -> RuntimeResult { + let capabilities = client.capabilities().await?; + capabilities.validate().map_err(RuntimeError::Protocol)?; + let missing_lifecycle = [ + RuntimeFeature::DurableIdentity, + RuntimeFeature::Stop, + RuntimeFeature::Remove, + ] + .into_iter() + .filter(|feature| !capabilities.supports_feature(*feature)) + .map(|feature| format!("feature:{feature:?}")) + .collect::>(); + if !missing_lifecycle.is_empty() { + return Err(RuntimeError::UnsupportedCapabilities(missing_lifecycle)); + } + fixture + .base_case() + .validate() + .map_err(RuntimeError::InvalidRequest)?; + for spec in fixture.base_case().specifications() { + let missing = capabilities + .missing_for(spec) + .map_err(RuntimeError::InvalidRequest)?; + if !missing.is_empty() { + return Err(RuntimeError::UnsupportedCapabilities(missing)); + } + } + + let required = required_runtime_profiles(&capabilities)?; + let mut selected = fixture.available_profiles(); + selected.insert(RuntimeConformanceProfile::Base); + let missing_profiles = required.difference(&selected).copied().collect::>(); + if !missing_profiles.is_empty() { + return Err(RuntimeError::Protocol(format!( + "conformance fixture omits required profiles: {}", + missing_profiles + .iter() + .map(|profile| profile.as_str()) + .collect::>() + .join(", ") + ))); + } + selected.extend(required); + + let inventory_before = fixture.inventory().await?; + let execution: RuntimeResult<( + RuntimeBaseConformanceReport, + Vec, + )> = async { + let base = verify_runtime_base(client, fixture.base_case()).await?; + let mut evidence = vec![base_evidence(&capabilities)?]; + for profile in selected + .iter() + .copied() + .filter(|profile| *profile != RuntimeConformanceProfile::Base) + { + let actual = fixture.run_profile(client, &capabilities, profile).await?; + validate_evidence(&capabilities, profile, &actual)?; + evidence.push(actual); + } + Ok((base, evidence)) + } + .await; + + let cleanup = fixture.cleanup().await; + let inventory_after = fixture.inventory().await; + cleanup?; + let inventory_after = inventory_after?; + if inventory_after != inventory_before { + return Err(RuntimeError::Protocol(format!( + "conformance cleanup changed provider inventory: before={:?}, after={:?}", + inventory_before.entries, inventory_after.entries + ))); + } + let (base, profiles) = execution?; + Ok(RuntimeConformanceSuiteReport { + base, + profiles, + inventory_before, + inventory_after, + }) +} + +fn validate_evidence( + capabilities: &RuntimeCapabilities, + profile: RuntimeConformanceProfile, + evidence: &RuntimeConformanceProfileEvidence, +) -> RuntimeResult<()> { + if evidence.profile != profile { + return Err(RuntimeError::Protocol(format!( + "conformance {} fixture returned {} evidence", + profile.as_str(), + evidence.profile.as_str() + ))); + } + let required = runtime_profile_requirements(capabilities, profile)?; + let missing_cases = required + .case_ids + .difference(&evidence.case_ids) + .cloned() + .collect::>(); + let missing_claims = required + .capability_claims + .difference(&evidence.capability_claims) + .cloned() + .collect::>(); + if !missing_cases.is_empty() || !missing_claims.is_empty() { + return Err(RuntimeError::Protocol(format!( + "conformance {} evidence is incomplete: missing cases {:?}, missing claims {:?}", + profile.as_str(), + missing_cases, + missing_claims + ))); + } + Ok(()) +} + +fn base_evidence( + capabilities: &RuntimeCapabilities, +) -> RuntimeResult { + let required = runtime_profile_requirements(capabilities, RuntimeConformanceProfile::Base)?; + Ok(RuntimeConformanceProfileEvidence { + profile: required.profile, + case_ids: required.case_ids, + capability_claims: required.capability_claims, + }) +} + +fn base_case_ids(profile: RuntimeConformanceProfile) -> &'static [&'static str] { + match profile { + RuntimeConformanceProfile::Base => &[ + "BASE-TASK-SUCCESS", + "BASE-TASK-FAILURE", + "BASE-TASK-TIMEOUT", + "BASE-SERVICE-LIFECYCLE", + "BASE-EXACT-REPLAY", + "BASE-GENERATION-CONFLICT", + "BASE-TOMBSTONE", + ], + RuntimeConformanceProfile::Recovery => &[ + "RECOVERY-CREATE-BEFORE-ACK", + "RECOVERY-CLIENT-RESTART", + "RECOVERY-PROVIDER-RESTART", + "RECOVERY-EXTERNAL-DELETION", + "RECOVERY-SAME-GENERATION-REPLACEMENT", + "RECOVERY-DUPLICATE-DETECTION", + ], + RuntimeConformanceProfile::Networking => &[], + RuntimeConformanceProfile::Mounts => &["MOUNT-READ-ONLY", "MOUNT-CLEANUP"], + RuntimeConformanceProfile::Health => &[ + "HEALTH-THRESHOLD-TRANSITION", + "HEALTH-PROBE-TIMEOUT", + "HEALTH-START-PERIOD", + "HEALTH-UNHEALTHY-EXIT", + ], + RuntimeConformanceProfile::Resources => &[], + RuntimeConformanceProfile::Logs => &[ + "LOG-STREAM-FILTER", + "LOG-TOTAL-ORDER", + "LOG-CURSOR-RESUME", + "LOG-SAME-TIMESTAMP", + "LOG-LIMIT", + "LOG-ROTATION-GAP", + "LOG-RETENTION", + "LOG-LARGE-RECORD", + ], + RuntimeConformanceProfile::Exec => &[ + "EXEC-STATE-POLICY", + "EXEC-TIMEOUT-REPLAY", + "EXEC-EXIT-CODE", + "EXEC-OUTPUT-BOUNDS", + "EXEC-OUTPUT-TRUNCATION", + "EXEC-IDENTITY-GENERATION-BINDING", + ], + RuntimeConformanceProfile::Security => &[ + "SECURITY-DIGEST-PINNING", + "SECURITY-METADATA-TAMPER", + "SECURITY-NAMESPACE-SEPARATION", + "SECURITY-LEAST-PRIVILEGE", + "SECURITY-HOSTILE-INPUT", + ], + RuntimeConformanceProfile::Outputs => &["OUTPUT-EXACT-BOUNDED", "OUTPUT-DIGEST-BINDING"], + RuntimeConformanceProfile::Evidence => &[ + "EVIDENCE-SPEC-BINDING", + "EVIDENCE-SEMANTICS-PROFILE-BINDING", + ], + } +} + +fn extend_capability_case_ids( + capabilities: &RuntimeCapabilities, + profile: RuntimeConformanceProfile, + case_ids: &mut BTreeSet, +) { + match profile { + RuntimeConformanceProfile::Networking => { + for mode in &capabilities.network_modes { + match mode { + NetworkMode::None => { + case_ids.insert("NETWORK-MODE-NONE".into()); + case_ids.insert("NETWORK-OUTBOUND-DENIED".into()); + } + NetworkMode::Outbound => { + case_ids.insert("NETWORK-MODE-OUTBOUND".into()); + case_ids.insert("NETWORK-OUTBOUND-ALLOWED".into()); + } + NetworkMode::Service => { + case_ids.insert("NETWORK-MODE-SERVICE".into()); + case_ids.insert("NETWORK-PROTOCOL-TCP".into()); + case_ids.insert("NETWORK-PROTOCOL-UDP".into()); + case_ids.insert("NETWORK-LOOPBACK-PUBLICATION".into()); + case_ids.insert("NETWORK-PORT-COLLISION".into()); + } + } + } + } + RuntimeConformanceProfile::Mounts => { + for kind in &capabilities.mount_kinds { + case_ids.insert( + match kind { + MountKind::Artifact => "MOUNT-ARTIFACT-BEHAVIOR", + MountKind::Volume => "MOUNT-VOLUME-PERSISTENCE", + MountKind::Tmpfs => "MOUNT-TMPFS-ISOLATION", + } + .into(), + ); + } + } + RuntimeConformanceProfile::Health => { + for kind in &capabilities.health_check_kinds { + case_ids.insert( + match kind { + HealthCheckKind::Http => "HEALTH-PROBE-HTTP", + HealthCheckKind::Tcp => "HEALTH-PROBE-TCP", + HealthCheckKind::Command => "HEALTH-PROBE-COMMAND", + } + .into(), + ); + } + } + RuntimeConformanceProfile::Resources => { + for control in &capabilities.resource_controls { + let (configuration, behavior) = match control { + ResourceControl::Cpu => ("RESOURCE-CPU-CONFIG", "RESOURCE-CPU-BEHAVIOR"), + ResourceControl::Memory => { + ("RESOURCE-MEMORY-CONFIG", "RESOURCE-MEMORY-BEHAVIOR") + } + ResourceControl::Pids => ("RESOURCE-PIDS-CONFIG", "RESOURCE-PIDS-BEHAVIOR"), + ResourceControl::EphemeralStorage => ( + "RESOURCE-EPHEMERAL-STORAGE-CONFIG", + "RESOURCE-EPHEMERAL-STORAGE-BEHAVIOR", + ), + ResourceControl::ExecutionTimeout => ( + "RESOURCE-EXECUTION-TIMEOUT-CONFIG", + "RESOURCE-EXECUTION-TIMEOUT-BEHAVIOR", + ), + }; + case_ids.insert(configuration.into()); + case_ids.insert(behavior.into()); + } + } + RuntimeConformanceProfile::Base + | RuntimeConformanceProfile::Recovery + | RuntimeConformanceProfile::Logs + | RuntimeConformanceProfile::Exec + | RuntimeConformanceProfile::Security + | RuntimeConformanceProfile::Outputs + | RuntimeConformanceProfile::Evidence => {} + } +} + +fn capability_claims( + capabilities: &RuntimeCapabilities, + profile: RuntimeConformanceProfile, +) -> BTreeSet { + match profile { + RuntimeConformanceProfile::Base => capabilities + .unit_classes + .iter() + .map(|class| format!("unit_class:{class:?}")) + .collect(), + RuntimeConformanceProfile::Recovery => BTreeSet::from(["feature:DurableIdentity".into()]), + RuntimeConformanceProfile::Networking => capabilities + .network_modes + .iter() + .map(|mode| format!("network_mode:{mode:?}")) + .collect(), + RuntimeConformanceProfile::Mounts => capabilities + .mount_kinds + .iter() + .map(|kind| format!("mount_kind:{kind:?}")) + .collect(), + RuntimeConformanceProfile::Health => capabilities + .health_check_kinds + .iter() + .map(|kind| format!("health_check:{kind:?}")) + .collect(), + RuntimeConformanceProfile::Resources => capabilities + .resource_controls + .iter() + .map(|control| format!("resource_control:{control:?}")) + .collect(), + RuntimeConformanceProfile::Logs => BTreeSet::from(["feature:Logs".into()]), + RuntimeConformanceProfile::Exec => BTreeSet::from(["feature:Exec".into()]), + RuntimeConformanceProfile::Security => { + let mut claims = capabilities + .isolation_levels + .iter() + .map(|level| format!("isolation:{level:?}")) + .collect::>(); + claims.insert("feature:DurableIdentity".into()); + if capabilities.supports_feature(RuntimeFeature::SecretReferences) { + claims.insert("feature:SecretReferences".into()); + } + claims + } + RuntimeConformanceProfile::Outputs => BTreeSet::from(["feature:OutputArtifacts".into()]), + RuntimeConformanceProfile::Evidence => [RuntimeFeature::Usage, RuntimeFeature::Attestation] + .into_iter() + .filter(|feature| capabilities.supports_feature(*feature)) + .map(|feature| format!("feature:{feature:?}")) + .collect(), + } +} diff --git a/src/contract/artifact.rs b/src/contract/artifact.rs index 68c1890..103a4e2 100644 --- a/src/contract/artifact.rs +++ b/src/contract/artifact.rs @@ -26,6 +26,7 @@ impl ArtifactRef { pub struct RuntimeOutputArtifact { pub name: String, pub artifact: ArtifactRef, + pub size_bytes: u64, } impl RuntimeOutputArtifact { diff --git a/src/contract/capabilities.rs b/src/contract/capabilities.rs index 4381618..3668d6f 100644 --- a/src/contract/capabilities.rs +++ b/src/contract/capabilities.rs @@ -1,6 +1,7 @@ use super::{ HealthCheckKind, IsolationLevel, MountKind, NetworkMode, RuntimeUnitClass, RuntimeUnitSpec, }; +use crate::ProviderId; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; @@ -25,6 +26,7 @@ pub enum RuntimeFeature { Usage, Attestation, SecretReferences, + OutputArtifacts, } /// Structured, provider-reported capabilities. Product-specific support @@ -33,7 +35,7 @@ pub enum RuntimeFeature { #[serde(deny_unknown_fields)] pub struct RuntimeCapabilities { pub schema: String, - pub provider_id: String, + pub provider_id: ProviderId, pub provider_build: String, pub unit_classes: Vec, pub artifact_media_types: Vec, @@ -46,7 +48,7 @@ pub struct RuntimeCapabilities { } impl RuntimeCapabilities { - pub const SCHEMA: &'static str = "a3s.runtime.capabilities.v2"; + pub const SCHEMA: &'static str = "a3s.runtime.capabilities.v3"; pub fn validate(&self) -> Result<(), String> { if self.schema != Self::SCHEMA { @@ -55,7 +57,6 @@ impl RuntimeCapabilities { self.schema )); } - super::validate_id("provider_id", &self.provider_id, 64)?; super::validate_nonempty("provider_build", &self.provider_build, 255)?; if self.unit_classes.is_empty() || self.artifact_media_types.is_empty() @@ -116,12 +117,18 @@ impl RuntimeCapabilities { ResourceControl::Cpu, ResourceControl::Memory, ResourceControl::Pids, - ResourceControl::EphemeralStorage, ] { if !self.resource_controls.contains(&required) { missing.push(format!("resource_control:{required:?}")); } } + if spec.resources.ephemeral_storage_bytes.is_some() + && !self + .resource_controls + .contains(&ResourceControl::EphemeralStorage) + { + missing.push("resource_control:EphemeralStorage".into()); + } if spec.resources.execution_timeout_ms.is_some() && !self .resource_controls @@ -135,6 +142,14 @@ impl RuntimeCapabilities { if !spec.secrets.is_empty() && !self.supports_feature(RuntimeFeature::SecretReferences) { missing.push("feature:SecretReferences".into()); } + if !spec.outputs.is_empty() && !self.supports_feature(RuntimeFeature::OutputArtifacts) { + missing.push("feature:OutputArtifacts".into()); + } + if spec.isolation == IsolationLevel::Confidential + && !self.supports_feature(RuntimeFeature::Attestation) + { + missing.push("feature:Attestation".into()); + } missing.sort(); missing.dedup(); Ok(missing) diff --git a/src/contract/observation.rs b/src/contract/observation.rs index 58c886f..66bde87 100644 --- a/src/contract/observation.rs +++ b/src/contract/observation.rs @@ -1,4 +1,6 @@ -use super::{ArtifactRef, RuntimeOutputArtifact, RuntimeUnitClass, RuntimeUnitSpec}; +use super::{ + ArtifactRef, IsolationLevel, RuntimeOutputArtifact, RuntimeUnitClass, RuntimeUnitSpec, +}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; @@ -117,7 +119,7 @@ pub struct RuntimeObservation { } impl RuntimeObservation { - pub const SCHEMA: &'static str = "a3s.runtime.observation.v1"; + pub const SCHEMA: &'static str = "a3s.runtime.observation.v2"; pub(crate) fn accepted(spec: &RuntimeUnitSpec, observed_at_ms: u64) -> Result { Ok(Self { @@ -227,6 +229,40 @@ impl RuntimeObservation { { return Err("Runtime observation does not match the unit specification".into()); } + if self.state == RuntimeUnitState::Succeeded { + if self.outputs.len() != spec.outputs.len() { + return Err("succeeded Task did not report the exact requested outputs".into()); + } + for expected in &spec.outputs { + let output = self + .outputs + .iter() + .find(|output| output.name == expected.name) + .ok_or_else(|| format!("succeeded Task omitted output {:?}", expected.name))?; + if output.artifact.media_type != expected.media_type { + return Err(format!( + "output {:?} media type does not match its specification", + expected.name + )); + } + if output.size_bytes > expected.max_bytes { + return Err(format!( + "output {:?} exceeds its maximum size", + expected.name + )); + } + } + } else if !self.outputs.is_empty() { + return Err("only a succeeded Task may report outputs".into()); + } + if spec.isolation == IsolationLevel::Confidential + && self.provider_resource_id.is_some() + && self.provider_attestation.is_none() + { + return Err( + "provider-backed confidential Runtime observation requires attestation".into(), + ); + } Ok(()) } @@ -252,22 +288,34 @@ impl RuntimeObservation { #[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] pub enum RuntimeInspection { Found { + schema: String, observation: Box, }, NotFound { + schema: String, unit_id: String, last_generation: Option, }, } impl RuntimeInspection { + pub const SCHEMA: &'static str = "a3s.runtime.inspection.v1"; + pub fn validate(&self) -> Result<(), String> { match self { - Self::Found { observation } => observation.validate(), + Self::Found { + schema, + observation, + } => { + validate_inspection_schema(schema)?; + observation.validate() + } Self::NotFound { + schema, unit_id, last_generation, } => { + validate_inspection_schema(schema)?; super::validate_id("unit_id", unit_id, 512)?; if *last_generation == Some(0) { return Err("last_generation must be positive when present".into()); @@ -277,3 +325,10 @@ impl RuntimeInspection { } } } + +fn validate_inspection_schema(schema: &str) -> Result<(), String> { + if schema != RuntimeInspection::SCHEMA { + return Err(format!("unsupported Runtime inspection schema {schema:?}")); + } + Ok(()) +} diff --git a/src/contract/protocol.rs b/src/contract/protocol.rs index 534db8b..a5340e6 100644 --- a/src/contract/protocol.rs +++ b/src/contract/protocol.rs @@ -106,6 +106,7 @@ pub enum RuntimeLogStream { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeLogQuery { + pub schema: String, pub unit_id: String, pub generation: u64, pub cursor: Option, @@ -114,7 +115,15 @@ pub struct RuntimeLogQuery { } impl RuntimeLogQuery { + pub const SCHEMA: &'static str = "a3s.runtime.log-query.v1"; + pub fn validate(&self) -> Result<(), String> { + if self.schema != Self::SCHEMA { + return Err(format!( + "unsupported Runtime log query schema {:?}", + self.schema + )); + } super::validate_id("unit_id", &self.unit_id, 512)?; if self.generation == 0 || self.limit == 0 || self.limit > 10_000 { return Err("log generation or limit is invalid".into()); @@ -133,6 +142,7 @@ impl RuntimeLogQuery { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeLogChunk { + pub schema: String, pub cursor: String, pub sequence: u64, pub observed_at_ms: u64, @@ -141,7 +151,15 @@ pub struct RuntimeLogChunk { } impl RuntimeLogChunk { + pub const SCHEMA: &'static str = "a3s.runtime.log-chunk.v1"; + pub fn validate(&self) -> Result<(), String> { + if self.schema != Self::SCHEMA { + return Err(format!( + "unsupported Runtime log chunk schema {:?}", + self.schema + )); + } super::validate_nonempty("log cursor", &self.cursor, 1024)?; if self.data.len() > 1024 * 1024 { return Err("log chunk exceeds one MiB".into()); @@ -153,19 +171,30 @@ impl RuntimeLogChunk { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeExecRequest { + pub schema: String, pub request_id: String, pub unit_id: String, pub generation: u64, pub command: Vec, pub timeout_ms: u64, + pub deadline_at_ms: Option, } impl RuntimeExecRequest { + pub const SCHEMA: &'static str = "a3s.runtime.exec-request.v1"; + pub fn validate(&self) -> Result<(), String> { + if self.schema != Self::SCHEMA { + return Err(format!( + "unsupported Runtime exec request schema {:?}", + self.schema + )); + } super::validate_id("request_id", &self.request_id, 512)?; super::validate_id("unit_id", &self.unit_id, 512)?; if self.generation == 0 || self.timeout_ms == 0 + || self.deadline_at_ms == Some(0) || self.command.is_empty() || self.command.len() > 256 || self @@ -177,11 +206,16 @@ impl RuntimeExecRequest { } Ok(()) } + + pub fn digest(&self) -> Result { + canonical_digest(self, self.validate()) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeExecResult { + pub schema: String, pub request_id: String, pub observation: RuntimeObservation, pub exit_code: i32, @@ -191,7 +225,15 @@ pub struct RuntimeExecResult { } impl RuntimeExecResult { + pub const SCHEMA: &'static str = "a3s.runtime.exec-result.v1"; + pub fn validate(&self) -> Result<(), String> { + if self.schema != Self::SCHEMA { + return Err(format!( + "unsupported Runtime exec result schema {:?}", + self.schema + )); + } super::validate_id("request_id", &self.request_id, 512)?; self.observation.validate()?; if self.stdout.len() > 16 * 1024 * 1024 || self.stderr.len() > 16 * 1024 * 1024 { diff --git a/src/contract/resource.rs b/src/contract/resource.rs index 7e10ffd..36d5972 100644 --- a/src/contract/resource.rs +++ b/src/contract/resource.rs @@ -15,19 +15,20 @@ pub struct ResourceLimits { pub cpu_millis: u64, pub memory_bytes: u64, pub pids: u32, - pub ephemeral_storage_bytes: u64, + /// Optional writable-layer quota. Providers must advertise + /// `ResourceControl::EphemeralStorage` before accepting this limit. + pub ephemeral_storage_bytes: Option, /// Required for finite Tasks and forbidden for long-running Services. pub execution_timeout_ms: Option, } impl ResourceLimits { pub(crate) fn validate(&self) -> Result<(), String> { - if self.cpu_millis == 0 - || self.memory_bytes == 0 - || self.pids == 0 - || self.ephemeral_storage_bytes == 0 - { - return Err("all Runtime resource limits must be positive".into()); + if self.cpu_millis == 0 || self.memory_bytes == 0 || self.pids == 0 { + return Err("required Runtime resource limits must be positive".into()); + } + if self.ephemeral_storage_bytes == Some(0) { + return Err("ephemeral_storage_bytes must be positive when present".into()); } if self.execution_timeout_ms == Some(0) { return Err("execution_timeout_ms must be positive when present".into()); diff --git a/src/contract/unit.rs b/src/contract/unit.rs index f260b6c..b39bbb0 100644 --- a/src/contract/unit.rs +++ b/src/contract/unit.rs @@ -226,7 +226,7 @@ pub struct RuntimeUnitSpec { } impl RuntimeUnitSpec { - pub const SCHEMA: &'static str = "a3s.runtime.unit-spec.v1"; + pub const SCHEMA: &'static str = "a3s.runtime.unit-spec.v2"; pub fn validate(&self) -> Result<(), String> { if self.schema != Self::SCHEMA { @@ -324,7 +324,7 @@ mod tests { cpu_millis: 500, memory_bytes: 128 * 1024 * 1024, pids: 128, - ephemeral_storage_bytes: 1024 * 1024 * 1024, + ephemeral_storage_bytes: Some(1024 * 1024 * 1024), execution_timeout_ms: timeout, } } diff --git a/src/driver.rs b/src/driver.rs index f4a28b7..cc0fa30 100644 --- a/src/driver.rs +++ b/src/driver.rs @@ -3,7 +3,7 @@ use crate::contract::{ RuntimeInspection, RuntimeLogChunk, RuntimeLogQuery, RuntimeObservation, RuntimeRemoval, RuntimeUnitSpec, }; -use crate::{RuntimeResult, RuntimeUnitRecord}; +use crate::{ProviderId, RuntimeResult, RuntimeUnitRecord}; use async_trait::async_trait; /// Provider-specific primitive used by `ManagedRuntimeClient`. @@ -11,8 +11,14 @@ use async_trait::async_trait; /// Drivers do not own request idempotency or durable shared state. `apply`, /// `stop`, and `remove` must nevertheless be safe to reattach after an /// ambiguous transport failure using the stable unit identity and generation. +/// A successful `apply` must also retire every older provider generation for +/// the unit before returning, leaving exactly one managed provider resource. +/// If that handoff is interrupted, an exact retry must discover the partially +/// created current generation and finish the same reconciliation. #[async_trait] pub trait RuntimeDriver: Send + Sync { + fn provider_id(&self) -> &ProviderId; + async fn capabilities(&self) -> RuntimeResult; async fn apply( @@ -41,6 +47,15 @@ pub trait RuntimeDriver: Send + Sync { query: &RuntimeLogQuery, ) -> RuntimeResult>; + /// Executes one durably identified request within its original budget. + /// + /// `ManagedRuntimeClient` always supplies `request.deadline_at_ms` as the + /// effective absolute deadline captured by the first reservation: the + /// smaller of that attempt's `timeout_ms` window and any caller-provided + /// absolute deadline. A pending replay receives the same persisted value, + /// so a driver must not restart or extend the execution window. Drivers may + /// enforce a shorter provider-specific timeout and must deduplicate or + /// reattach the stable request ID after an ambiguous result. async fn exec( &self, unit: &RuntimeUnitRecord, diff --git a/src/error.rs b/src/error.rs index a97c042..aab5d64 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,6 +10,8 @@ pub enum RuntimeError { NotFound { unit_id: String }, #[error("Runtime request {request_id:?} conflicts with a prior request")] RequestConflict { request_id: String }, + #[error("Runtime request {request_id:?} for unit {unit_id:?} was not found")] + RequestNotFound { unit_id: String, request_id: String }, #[error( "Runtime unit {unit_id:?} rejected stale generation {requested}; current generation is {current}" )] diff --git a/src/lib.rs b/src/lib.rs index 85f3ab8..8e0f2e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,13 +14,20 @@ pub mod contract; pub use client::RuntimeClient; pub use clock::{RuntimeClock, SystemRuntimeClock}; -pub use conformance::{verify_runtime_provider, RuntimeConformanceCase, RuntimeConformanceReport}; +pub use conformance::{ + required_runtime_profiles, runtime_profile_requirements, verify_runtime_base, + verify_runtime_profiles, verify_runtime_provider, RuntimeBaseConformanceCase, + RuntimeBaseConformanceReport, RuntimeConformanceCase, RuntimeConformanceFixture, + RuntimeConformanceInventory, RuntimeConformanceProfile, RuntimeConformanceProfileEvidence, + RuntimeConformanceProfileRequirements, RuntimeConformanceReport, RuntimeConformanceSuiteReport, +}; pub use driver::RuntimeDriver; pub use error::{RuntimeError, RuntimeResult}; pub use managed::ManagedRuntimeClient; pub use provider::ProviderId; pub use registry::{RuntimeClientRegistry, RuntimeProviderFactory}; pub use state::{ - FileRuntimeStateStore, RuntimeActionKind, RuntimeRequestKind, RuntimeRequestReceipt, - RuntimeRequestState, RuntimeStateReservation, RuntimeStateStore, RuntimeUnitRecord, + FileRuntimeStateStore, RuntimeActionKind, RuntimeOperationLease, RuntimeRequestKind, + RuntimeRequestReceipt, RuntimeRequestState, RuntimeStateReservation, RuntimeStateStore, + RuntimeUnitRecord, }; diff --git a/src/managed.rs b/src/managed.rs index 1a26fb9..85b7c36 100644 --- a/src/managed.rs +++ b/src/managed.rs @@ -4,11 +4,14 @@ use crate::contract::{ RuntimeRemoval, RuntimeUnitState, }; use crate::{ - RuntimeActionKind, RuntimeClient, RuntimeClock, RuntimeDriver, RuntimeError, RuntimeResult, + RuntimeActionKind, RuntimeClient, RuntimeClock, RuntimeDriver, RuntimeError, + RuntimeRequestKind, RuntimeRequestReceipt, RuntimeRequestState, RuntimeResult, RuntimeStateStore, SystemRuntimeClock, }; use async_trait::async_trait; +use std::future::Future; use std::sync::Arc; +use std::time::Duration; /// Shared durable lifecycle implementation used by provider integrations. pub struct ManagedRuntimeClient { @@ -37,6 +40,13 @@ impl ManagedRuntimeClient { async fn checked_capabilities(&self) -> RuntimeResult { let capabilities = self.driver.capabilities().await?; capabilities.validate().map_err(RuntimeError::Protocol)?; + if &capabilities.provider_id != self.driver.provider_id() { + return Err(RuntimeError::Protocol(format!( + "Runtime driver {:?} reported capabilities for {:?}", + self.driver.provider_id().as_str(), + capabilities.provider_id.as_str() + ))); + } Ok(capabilities) } @@ -48,6 +58,88 @@ impl ManagedRuntimeClient { } Ok(()) } + + async fn bounded( + &self, + deadline_at_ms: Option, + stage: &'static str, + future: F, + ) -> RuntimeResult + where + T: Send, + F: Future> + Send, + { + let Some(deadline_at_ms) = deadline_at_ms else { + return future.await; + }; + let now_ms = self.clock.now_ms(); + let Some(remaining_ms) = deadline_at_ms.checked_sub(now_ms) else { + return Err(RuntimeError::DeadlineExceeded(format!( + "request expired before {stage}" + ))); + }; + if remaining_ms == 0 { + return Err(RuntimeError::DeadlineExceeded(format!( + "request expired before {stage}" + ))); + } + tokio::time::timeout(Duration::from_millis(remaining_ms), future) + .await + .map_err(|_| { + RuntimeError::DeadlineExceeded(format!("request deadline elapsed during {stage}")) + })? + } + + fn exec_deadline(&self, request: &RuntimeExecRequest, started_at_ms: u64) -> u64 { + let relative = started_at_ms.saturating_add(request.timeout_ms); + request + .deadline_at_ms + .map_or(relative, |absolute| absolute.min(relative)) + } + + async fn matching_receipt( + &self, + unit_id: &str, + generation: u64, + request_id: &str, + kind: RuntimeRequestKind, + request_digest: &str, + ) -> RuntimeResult> { + let receipt = match self.state.load_request(unit_id, request_id).await { + Ok(receipt) => receipt, + Err(RuntimeError::RequestNotFound { .. }) => return Ok(None), + Err(error) => return Err(error), + }; + receipt.validate().map_err(RuntimeError::Protocol)?; + if receipt.unit_id != unit_id || receipt.request_id != request_id { + return Err(RuntimeError::Protocol( + "Runtime request receipt storage key mismatch".into(), + )); + } + if receipt.generation != generation + || receipt.kind != kind + || receipt.request_digest != request_digest + { + return Err(RuntimeError::RequestConflict { + request_id: request_id.into(), + }); + } + Ok(Some(receipt)) + } + + async fn completed_replay( + &self, + unit_id: &str, + generation: u64, + request_id: &str, + kind: RuntimeRequestKind, + request_digest: &str, + ) -> RuntimeResult> { + Ok(self + .matching_receipt(unit_id, generation, request_id, kind, request_digest) + .await? + .filter(|receipt| receipt.state == RuntimeRequestState::Completed)) + } } #[async_trait] @@ -58,8 +150,43 @@ impl RuntimeClient for ManagedRuntimeClient { async fn apply(&self, request: &RuntimeApplyRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; + let request_digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + if self + .completed_replay( + &request.spec.unit_id, + request.spec.generation, + &request.request_id, + RuntimeRequestKind::Apply, + &request_digest, + ) + .await? + .is_some() + { + let _lease = self + .state + .acquire_operation_lease(&request.spec.unit_id) + .await?; + let reservation = self + .state + .reserve_apply(request, self.clock.now_ms()) + .await?; + if reservation.dispatch { + return Err(RuntimeError::Protocol( + "completed apply receipt regressed to pending".into(), + )); + } + return reservation.receipt.observation.ok_or_else(|| { + RuntimeError::Protocol("completed apply receipt has no observation".into()) + }); + } self.check_deadline(request.deadline_at_ms)?; - let capabilities = self.checked_capabilities().await?; + let capabilities = self + .bounded( + request.deadline_at_ms, + "capability query", + self.checked_capabilities(), + ) + .await?; let missing = capabilities .missing_for(&request.spec) .map_err(RuntimeError::InvalidRequest)?; @@ -67,6 +194,15 @@ impl RuntimeClient for ManagedRuntimeClient { return Err(RuntimeError::UnsupportedCapabilities(missing)); } + let _lease = self + .bounded( + request.deadline_at_ms, + "operation lease wait", + self.state.acquire_operation_lease(&request.spec.unit_id), + ) + .await?; + self.check_deadline(request.deadline_at_ms)?; + let reservation = self .state .reserve_apply(request, self.clock.now_ms()) @@ -78,17 +214,17 @@ impl RuntimeClient for ManagedRuntimeClient { } let observation = self - .driver - .apply(&request.spec, &reservation.record.observation) + .bounded( + request.deadline_at_ms, + "provider apply", + self.driver + .apply(&request.spec, &reservation.record.observation), + ) .await?; observation .validate_against(&request.spec) .map_err(RuntimeError::Protocol)?; - if observation.state == RuntimeUnitState::Accepted { - return Err(RuntimeError::Protocol( - "provider apply did not advance the accepted observation".into(), - )); - } + ensure_apply_result(&request.spec, &observation)?; Ok(self .state .update_observation(Some(&request.request_id), &observation) @@ -97,10 +233,12 @@ impl RuntimeClient for ManagedRuntimeClient { } async fn inspect(&self, unit_id: &str) -> RuntimeResult { + let _lease = self.state.acquire_operation_lease(unit_id).await?; let record = match self.state.load(unit_id).await { Ok(record) => record, Err(RuntimeError::NotFound { .. }) => { return Ok(RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), unit_id: unit_id.into(), last_generation: None, }); @@ -109,18 +247,22 @@ impl RuntimeClient for ManagedRuntimeClient { }; if record.removed_at_ms.is_some() { return Ok(RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), unit_id: unit_id.into(), last_generation: Some(record.spec.generation), }); } if record.observation.state.is_terminal() { return Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(record.observation), }); } - match self.driver.inspect(&record).await? { - RuntimeInspection::Found { observation } => { + let inspection = self.driver.inspect(&record).await?; + inspection.validate().map_err(RuntimeError::Protocol)?; + match inspection { + RuntimeInspection::Found { observation, .. } => { observation .validate_against(&record.spec) .map_err(RuntimeError::Protocol)?; @@ -129,6 +271,7 @@ impl RuntimeClient for ManagedRuntimeClient { .update_observation(None, observation.as_ref()) .await?; Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(record.observation), }) } @@ -142,6 +285,7 @@ impl RuntimeClient for ManagedRuntimeClient { unknown.failure = None; let record = self.state.update_observation(None, &unknown).await?; Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(record.observation), }) } @@ -150,46 +294,138 @@ impl RuntimeClient for ManagedRuntimeClient { async fn stop(&self, request: &RuntimeActionRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; + let request_digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + if self + .completed_replay( + &request.unit_id, + request.generation, + &request.request_id, + RuntimeRequestKind::Stop, + &request_digest, + ) + .await? + .is_some() + { + let _lease = self.state.acquire_operation_lease(&request.unit_id).await?; + let reservation = self + .state + .reserve_action(RuntimeActionKind::Stop, request, self.clock.now_ms()) + .await?; + if reservation.dispatch { + return Err(RuntimeError::Protocol( + "completed stop receipt regressed to pending".into(), + )); + } + return Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), + observation: Box::new(reservation.receipt.observation.ok_or_else(|| { + RuntimeError::Protocol("completed stop receipt has no observation".into()) + })?), + }); + } self.check_deadline(request.deadline_at_ms)?; - let capabilities = self.checked_capabilities().await?; + let capabilities = self + .bounded( + request.deadline_at_ms, + "capability query", + self.checked_capabilities(), + ) + .await?; if !capabilities.supports_feature(crate::contract::RuntimeFeature::Stop) { return Err(RuntimeError::UnsupportedCapabilities(vec![ "feature:Stop".into() ])); } + let _lease = self + .bounded( + request.deadline_at_ms, + "operation lease wait", + self.state.acquire_operation_lease(&request.unit_id), + ) + .await?; + self.check_deadline(request.deadline_at_ms)?; let reservation = self .state .reserve_action(RuntimeActionKind::Stop, request, self.clock.now_ms()) .await?; if !reservation.dispatch { return Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(reservation.receipt.observation.ok_or_else(|| { RuntimeError::Protocol("completed stop receipt has no observation".into()) })?), }); } - let observation = self.driver.stop(&reservation.record, request).await?; + let observation = self + .bounded( + request.deadline_at_ms, + "provider stop", + self.driver.stop(&reservation.record, request), + ) + .await?; observation .validate_against(&reservation.record.spec) .map_err(RuntimeError::Protocol)?; + ensure_stop_result(&reservation.record.observation, &observation)?; let record = self .state .update_observation(Some(&request.request_id), &observation) .await?; Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(record.observation), }) } async fn remove(&self, request: &RuntimeActionRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; + let request_digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + if self + .completed_replay( + &request.unit_id, + request.generation, + &request.request_id, + RuntimeRequestKind::Remove, + &request_digest, + ) + .await? + .is_some() + { + let _lease = self.state.acquire_operation_lease(&request.unit_id).await?; + let reservation = self + .state + .reserve_action(RuntimeActionKind::Remove, request, self.clock.now_ms()) + .await?; + if reservation.dispatch { + return Err(RuntimeError::Protocol( + "completed remove receipt regressed to pending".into(), + )); + } + return reservation.receipt.removal.ok_or_else(|| { + RuntimeError::Protocol("completed remove receipt has no removal".into()) + }); + } self.check_deadline(request.deadline_at_ms)?; - let capabilities = self.checked_capabilities().await?; + let capabilities = self + .bounded( + request.deadline_at_ms, + "capability query", + self.checked_capabilities(), + ) + .await?; if !capabilities.supports_feature(crate::contract::RuntimeFeature::Remove) { return Err(RuntimeError::UnsupportedCapabilities(vec![ "feature:Remove".into(), ])); } + let _lease = self + .bounded( + request.deadline_at_ms, + "operation lease wait", + self.state.acquire_operation_lease(&request.unit_id), + ) + .await?; + self.check_deadline(request.deadline_at_ms)?; let reservation = self .state .reserve_action(RuntimeActionKind::Remove, request, self.clock.now_ms()) @@ -199,7 +435,13 @@ impl RuntimeClient for ManagedRuntimeClient { RuntimeError::Protocol("completed remove receipt has no removal".into()) }); } - let removal = self.driver.remove(&reservation.record, request).await?; + let removal = self + .bounded( + request.deadline_at_ms, + "provider remove", + self.driver.remove(&reservation.record, request), + ) + .await?; removal.validate().map_err(RuntimeError::Protocol)?; if removal.request_id != request.request_id || removal.unit_id != request.unit_id @@ -221,8 +463,9 @@ impl RuntimeClient for ManagedRuntimeClient { "feature:Logs".into() ])); } + let _lease = self.state.acquire_operation_lease(&query.unit_id).await?; let record = self.state.load(&query.unit_id).await?; - ensure_active_generation(&record, query.generation)?; + ensure_current_generation(&record, query.generation)?; let chunks = self.driver.logs(&record, query).await?; for chunk in &chunks { chunk.validate().map_err(RuntimeError::Protocol)?; @@ -240,16 +483,89 @@ impl RuntimeClient for ManagedRuntimeClient { async fn exec(&self, request: &RuntimeExecRequest) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; - let capabilities = self.checked_capabilities().await?; + let request_digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + let operation_started_at_ms = self.clock.now_ms(); + let existing = self + .matching_receipt( + &request.unit_id, + request.generation, + &request.request_id, + RuntimeRequestKind::Exec, + &request_digest, + ) + .await?; + if existing + .as_ref() + .is_some_and(|receipt| receipt.state == RuntimeRequestState::Completed) + { + let _lease = self.state.acquire_operation_lease(&request.unit_id).await?; + let reservation = self + .state + .reserve_exec(request, operation_started_at_ms) + .await?; + if reservation.dispatch { + return Err(RuntimeError::Protocol( + "completed exec receipt regressed to pending".into(), + )); + } + return reservation.receipt.exec_result.ok_or_else(|| { + RuntimeError::Protocol("completed exec receipt has no result".into()) + }); + } + let deadline_at_ms = existing + .as_ref() + .and_then(|receipt| receipt.deadline_at_ms) + .unwrap_or_else(|| self.exec_deadline(request, operation_started_at_ms)); + let deadline_at_ms = Some(deadline_at_ms); + self.check_deadline(deadline_at_ms)?; + let capabilities = self + .bounded( + deadline_at_ms, + "capability query", + self.checked_capabilities(), + ) + .await?; if !capabilities.supports_feature(crate::contract::RuntimeFeature::Exec) { return Err(RuntimeError::UnsupportedCapabilities(vec![ "feature:Exec".into() ])); } - let record = self.state.load(&request.unit_id).await?; - ensure_active_generation(&record, request.generation)?; - let result = self.driver.exec(&record, request).await?; + let _lease = self + .bounded( + deadline_at_ms, + "operation lease wait", + self.state.acquire_operation_lease(&request.unit_id), + ) + .await?; + self.check_deadline(deadline_at_ms)?; + let reservation = self + .state + .reserve_exec(request, operation_started_at_ms) + .await?; + if !reservation.dispatch { + return reservation.receipt.exec_result.ok_or_else(|| { + RuntimeError::Protocol("completed exec receipt has no result".into()) + }); + } + let deadline_at_ms = reservation.receipt.deadline_at_ms.ok_or_else(|| { + RuntimeError::Protocol("pending exec receipt has no effective deadline".into()) + })?; + let deadline_at_ms = Some(deadline_at_ms); + self.check_deadline(deadline_at_ms)?; + let mut provider_request = request.clone(); + provider_request.deadline_at_ms = deadline_at_ms; + let result = self + .bounded( + deadline_at_ms, + "provider exec", + self.driver.exec(&reservation.record, &provider_request), + ) + .await?; result.validate().map_err(RuntimeError::Protocol)?; + result + .observation + .validate_against(&reservation.record.spec) + .map_err(RuntimeError::Protocol)?; if result.request_id != request.request_id || result.observation.unit_id != request.unit_id || result.observation.generation != request.generation @@ -258,11 +574,12 @@ impl RuntimeClient for ManagedRuntimeClient { "provider exec changed immutable request identity".into(), )); } + self.state.complete_exec(&result).await?; Ok(result) } } -fn ensure_active_generation( +fn ensure_current_generation( record: &crate::RuntimeUnitRecord, requested: u64, ) -> RuntimeResult<()> { @@ -286,3 +603,50 @@ fn ensure_active_generation( } Ok(()) } + +fn ensure_apply_result( + spec: &crate::contract::RuntimeUnitSpec, + observation: &RuntimeObservation, +) -> RuntimeResult<()> { + let allowed = match spec.class { + crate::contract::RuntimeUnitClass::Task => matches!( + observation.state, + RuntimeUnitState::Succeeded | RuntimeUnitState::Failed + ), + crate::contract::RuntimeUnitClass::Service => matches!( + observation.state, + RuntimeUnitState::Running + | RuntimeUnitState::Stopped + | RuntimeUnitState::Failed + | RuntimeUnitState::Unknown + ), + }; + if !allowed { + return Err(RuntimeError::Protocol(format!( + "provider apply returned invalid {:?} result {:?}", + spec.class, observation.state + ))); + } + if observation.provider_resource_id.is_none() || observation.provider_build.is_none() { + return Err(RuntimeError::Protocol( + "provider apply returned an observation without provider identity".into(), + )); + } + Ok(()) +} + +fn ensure_stop_result( + current: &RuntimeObservation, + observation: &RuntimeObservation, +) -> RuntimeResult<()> { + if observation.state == RuntimeUnitState::Stopped + || observation.state == RuntimeUnitState::Unknown + || current.state.is_terminal() && observation == current + { + return Ok(()); + } + Err(RuntimeError::Protocol(format!( + "provider stop returned nonterminal state {:?}", + observation.state + ))) +} diff --git a/src/registry.rs b/src/registry.rs index b66a4c2..9f6365d 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -1,12 +1,14 @@ use crate::{ProviderId, RuntimeClient, RuntimeError, RuntimeResult}; +use async_trait::async_trait; use std::collections::BTreeMap; use std::sync::Arc; /// Typed construction boundary for one Runtime provider implementation. +#[async_trait] pub trait RuntimeProviderFactory: Send + Sync { fn provider_id(&self) -> &ProviderId; - fn create(&self) -> RuntimeResult>; + async fn create(&self) -> RuntimeResult>; } /// Registry of provider factories. Selection policy belongs to the caller; @@ -36,8 +38,9 @@ impl RuntimeClientRegistry { self.factories.contains_key(provider) } - pub fn connect(&self, provider: &ProviderId) -> RuntimeResult> { - self.factories + pub async fn connect(&self, provider: &ProviderId) -> RuntimeResult> { + let client = self + .factories .get(provider) .ok_or_else(|| { RuntimeError::ProviderUnavailable(format!( @@ -46,5 +49,16 @@ impl RuntimeClientRegistry { )) })? .create() + .await?; + let capabilities = client.capabilities().await?; + capabilities.validate().map_err(RuntimeError::Protocol)?; + if &capabilities.provider_id != provider { + return Err(RuntimeError::Protocol(format!( + "Runtime provider factory {:?} created client reporting {:?}", + provider.as_str(), + capabilities.provider_id.as_str() + ))); + } + Ok(client) } } diff --git a/src/state/file.rs b/src/state/file.rs index e045ec2..76762cc 100644 --- a/src/state/file.rs +++ b/src/state/file.rs @@ -1,17 +1,25 @@ +#[cfg(test)] +use self::fs::test_failpoint; +use self::fs::{ + atomic_write, ensure_directory, io_error, owner_only_open, path_exists, read_optional_receipt, + read_optional_record, read_required_receipt, read_required_record, +}; use super::{ - RuntimeActionKind, RuntimeRequestKind, RuntimeRequestReceipt, RuntimeRequestState, - RuntimeStateReservation, RuntimeStateStore, RuntimeUnitRecord, + RuntimeActionKind, RuntimeOperationLease, RuntimeRequestKind, RuntimeRequestReceipt, + RuntimeRequestState, RuntimeStateReservation, RuntimeStateStore, RuntimeUnitRecord, }; use crate::contract::{ - RuntimeActionRequest, RuntimeApplyRequest, RuntimeObservation, RuntimeRemoval, RuntimeUnitState, + RuntimeActionRequest, RuntimeApplyRequest, RuntimeExecRequest, RuntimeExecResult, + RuntimeObservation, RuntimeRemoval, RuntimeUnitState, }; use crate::{RuntimeError, RuntimeResult}; use async_trait::async_trait; use fs2::FileExt; use sha2::{Digest, Sha256}; -use std::fs::{File, OpenOptions}; -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; +use std::fs::File; +use std::path::PathBuf; + +mod fs; #[derive(Debug, Clone)] pub struct FileRuntimeStateStore { @@ -23,6 +31,18 @@ impl FileRuntimeStateStore { Self { root: root.into() } } + fn acquire_operation_lease_sync(&self, unit_id: &str) -> RuntimeResult { + validate_unit_id(unit_id)?; + ensure_directory(&self.root)?; + let operations = self.root.join("operations"); + ensure_directory(&operations)?; + let path = operations.join(format!("{}.lock", storage_key(unit_id))); + let file = owner_only_open(&path, "Runtime operation lease")?; + file.lock_exclusive() + .map_err(io_error("lock Runtime operation"))?; + Ok(FileOperationLease(file)) + } + fn reserve_apply_sync( &self, request: RuntimeApplyRequest, @@ -30,20 +50,40 @@ impl FileRuntimeStateStore { ) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; let _lock = self.lock(&request.spec.unit_id)?; - let path = self.record_path(&request.spec.unit_id)?; - let mut record = if path.exists() { - read_record(&path)? - } else { - let record = - RuntimeUnitRecord::new(&request, now_ms).map_err(RuntimeError::Protocol)?; - atomic_write(&path, &record)?; - return reservation(record, &request.request_id); + let record_path = self.record_path(&request.spec.unit_id, true)?; + let mut receipt_path = + self.request_path(&request.spec.unit_id, &request.request_id, false)?; + let existing = read_optional_receipt(&receipt_path)?; + if let Some(receipt) = &existing { + let digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + ensure_same_request(receipt, RuntimeRequestKind::Apply, &digest)?; + ensure_receipt_target(receipt, &request.spec.unit_id, request.spec.generation)?; + } + + let stored_record = read_optional_record(&record_path)?; + if stored_record.is_none() + && existing + .as_ref() + .is_some_and(|receipt| receipt.state == RuntimeRequestState::Completed) + { + return Err(RuntimeError::Protocol( + "completed Runtime apply receipt has no unit record".into(), + )); + } + let mut record_changed = stored_record.is_none(); + let mut record = match stored_record { + Some(record) => record, + None => RuntimeUnitRecord::new(&request, now_ms).map_err(RuntimeError::Protocol)?, }; - if let Some(existing) = record.receipt(&request.request_id) { - let digest = request.digest().map_err(RuntimeError::InvalidRequest)?; - ensure_same_request(existing, RuntimeRequestKind::Apply, &digest)?; - return reservation(record, &request.request_id); + if let Some(receipt) = existing + .as_ref() + .filter(|receipt| receipt.state == RuntimeRequestState::Completed) + { + if reconcile_completed_observation(&mut record, receipt)? { + atomic_write(&record_path, &record, "state record")?; + } + return Ok(reservation(record, receipt.clone())); } let current_generation = record.spec.generation; @@ -55,6 +95,11 @@ impl FileRuntimeStateStore { }); } + let receipt_is_new = existing.is_none(); + let mut receipt = existing.unwrap_or( + RuntimeRequestReceipt::pending_apply(&request).map_err(RuntimeError::Protocol)?, + ); + if request.spec.generation == current_generation { if request .spec @@ -68,25 +113,33 @@ impl FileRuntimeStateStore { generation: request.spec.generation, }); } - let mut receipt = - RuntimeRequestReceipt::pending_apply(&request).map_err(RuntimeError::Protocol)?; - if record.observation.state != RuntimeUnitState::Accepted { + if !matches!( + record.observation.state, + RuntimeUnitState::Accepted | RuntimeUnitState::Unknown + ) { receipt.complete_with_observation(record.observation.clone()); + receipt.validate().map_err(RuntimeError::Protocol)?; } - record.requests.insert(request.request_id.clone(), receipt); } else { record.spec = request.spec.clone(); record.observation = RuntimeObservation::accepted(&request.spec, now_ms) .map_err(RuntimeError::Protocol)?; record.removed_at_ms = None; - record.requests.insert( - request.request_id.clone(), - RuntimeRequestReceipt::pending_apply(&request).map_err(RuntimeError::Protocol)?, - ); + record_changed = true; + } + + if receipt_is_new || receipt.state == RuntimeRequestState::Completed { + if receipt_is_new { + receipt_path = + self.request_path(&request.spec.unit_id, &request.request_id, true)?; + } + atomic_write(&receipt_path, &receipt, "request receipt")?; } record.validate().map_err(RuntimeError::Protocol)?; - atomic_write(&path, &record)?; - reservation(record, &request.request_id) + if record_changed { + atomic_write(&record_path, &record, "state record")?; + } + Ok(reservation(record, receipt)) } fn reserve_action_sync( @@ -97,32 +150,38 @@ impl FileRuntimeStateStore { ) -> RuntimeResult { request.validate().map_err(RuntimeError::InvalidRequest)?; let _lock = self.lock(&request.unit_id)?; - let path = self.record_path(&request.unit_id)?; - if !path.is_file() { - return Err(RuntimeError::NotFound { - unit_id: request.unit_id, - }); - } - let mut record = read_record(&path)?; - if let Some(existing) = record.receipt(&request.request_id) { + let record_path = self.record_path(&request.unit_id, false)?; + let mut record = read_required_record(&record_path, &request.unit_id)?; + let mut receipt_path = self.request_path(&request.unit_id, &request.request_id, false)?; + if let Some(receipt) = read_optional_receipt(&receipt_path)? { let digest = request.digest().map_err(RuntimeError::InvalidRequest)?; - ensure_same_request(existing, kind.into(), &digest)?; - return reservation(record, &request.request_id); - } - if request.generation < record.spec.generation { - return Err(RuntimeError::StaleGeneration { - unit_id: request.unit_id, - requested: request.generation, - current: record.spec.generation, - }); - } - if request.generation != record.spec.generation { - return Err(RuntimeError::GenerationConflict { - unit_id: request.unit_id, - generation: request.generation, - }); + ensure_same_request(&receipt, kind.into(), &digest)?; + ensure_receipt_target(&receipt, &request.unit_id, request.generation)?; + if receipt.state == RuntimeRequestState::Completed { + let changed = match receipt.kind { + RuntimeRequestKind::Stop => { + reconcile_completed_observation(&mut record, &receipt)? + } + RuntimeRequestKind::Remove => { + reconcile_completed_removal(&mut record, &receipt)? + } + _ => false, + }; + if changed { + atomic_write(&record_path, &record, "state record")?; + } + } else { + ensure_current_generation(&record, &request.unit_id, request.generation)?; + if record.removed_at_ms.is_some() { + return Err(RuntimeError::NotFound { + unit_id: request.unit_id, + }); + } + } + return Ok(reservation(record, receipt)); } + ensure_current_generation(&record, &request.unit_id, request.generation)?; let mut receipt = RuntimeRequestReceipt::pending_action(kind, &request) .map_err(RuntimeError::Protocol)?; match kind { @@ -149,26 +208,88 @@ impl FileRuntimeStateStore { } } } - record.requests.insert(request.request_id.clone(), receipt); - record.validate().map_err(RuntimeError::Protocol)?; - atomic_write(&path, &record)?; - reservation(record, &request.request_id) + receipt.validate().map_err(RuntimeError::Protocol)?; + receipt_path = self.request_path(&request.unit_id, &request.request_id, true)?; + atomic_write(&receipt_path, &receipt, "request receipt")?; + Ok(reservation(record, receipt)) + } + + fn reserve_exec_sync( + &self, + request: RuntimeExecRequest, + started_at_ms: u64, + ) -> RuntimeResult { + request.validate().map_err(RuntimeError::InvalidRequest)?; + let _lock = self.lock(&request.unit_id)?; + let record_path = self.record_path(&request.unit_id, false)?; + let mut record = read_required_record(&record_path, &request.unit_id)?; + let mut receipt_path = self.request_path(&request.unit_id, &request.request_id, false)?; + if let Some(receipt) = read_optional_receipt(&receipt_path)? { + let digest = request.digest().map_err(RuntimeError::InvalidRequest)?; + ensure_same_request(&receipt, RuntimeRequestKind::Exec, &digest)?; + ensure_receipt_target(&receipt, &request.unit_id, request.generation)?; + if receipt.state == RuntimeRequestState::Completed + && reconcile_completed_exec(&mut record, &receipt)? + { + atomic_write(&record_path, &record, "state record")?; + } else if receipt.state == RuntimeRequestState::Pending { + ensure_current_generation(&record, &request.unit_id, request.generation)?; + if record.removed_at_ms.is_some() { + return Err(RuntimeError::NotFound { + unit_id: request.unit_id, + }); + } + } + return Ok(reservation(record, receipt)); + } + + ensure_current_generation(&record, &request.unit_id, request.generation)?; + if record.removed_at_ms.is_some() { + return Err(RuntimeError::NotFound { + unit_id: request.unit_id, + }); + } + if record.observation.state != RuntimeUnitState::Running { + return Err(RuntimeError::InvalidRequest(format!( + "Runtime exec requires a running unit; {:?} is {:?}", + request.unit_id, record.observation.state + ))); + } + let receipt = RuntimeRequestReceipt::pending_exec(&request, started_at_ms) + .map_err(RuntimeError::Protocol)?; + receipt.validate().map_err(RuntimeError::Protocol)?; + receipt_path = self.request_path(&request.unit_id, &request.request_id, true)?; + atomic_write(&receipt_path, &receipt, "request receipt")?; + Ok(reservation(record, receipt)) } fn load_sync(&self, unit_id: &str) -> RuntimeResult { validate_unit_id(unit_id)?; let _lock = self.lock(unit_id)?; - let path = self.record_path(unit_id)?; - if !path.is_file() { - return Err(RuntimeError::NotFound { + let path = self.record_path(unit_id, false)?; + read_required_record(&path, unit_id) + } + + fn load_request_sync( + &self, + unit_id: &str, + request_id: &str, + ) -> RuntimeResult { + validate_unit_id(unit_id)?; + validate_request_id(request_id)?; + let _lock = self.lock(unit_id)?; + let path = self.request_path(unit_id, request_id, false)?; + let receipt = + read_optional_receipt(&path)?.ok_or_else(|| RuntimeError::RequestNotFound { unit_id: unit_id.into(), - }); - } - let record = read_record(&path)?; - if record.spec.unit_id != unit_id { - return Err(RuntimeError::Protocol("Runtime state key mismatch".into())); + request_id: request_id.into(), + })?; + if receipt.unit_id != unit_id || receipt.request_id != request_id { + return Err(RuntimeError::Protocol( + "Runtime request receipt storage key mismatch".into(), + )); } - Ok(record) + Ok(receipt) } fn update_observation_sync( @@ -178,63 +299,126 @@ impl FileRuntimeStateStore { ) -> RuntimeResult { observation.validate().map_err(RuntimeError::Protocol)?; let _lock = self.lock(&observation.unit_id)?; - let path = self.record_path(&observation.unit_id)?; - if !path.is_file() { - return Err(RuntimeError::NotFound { - unit_id: observation.unit_id, - }); - } - let mut record = read_record(&path)?; + let record_path = self.record_path(&observation.unit_id, false)?; + let mut record = read_required_record(&record_path, &observation.unit_id)?; if record.removed_at_ms.is_some() { return Err(RuntimeError::Protocol( "cannot update an explicitly removed Runtime unit".into(), )); } validate_transition(&record.observation, &observation, &record.spec)?; - record.observation = observation.clone(); + if let Some(request_id) = request_id { - let receipt = record - .receipt_mut(&request_id) - .map_err(RuntimeError::Protocol)?; - if receipt.kind == RuntimeRequestKind::Remove { + let receipt_path = self.request_path(&observation.unit_id, &request_id, false)?; + let mut receipt = + read_required_receipt(&receipt_path, &observation.unit_id, &request_id)?; + if !matches!( + receipt.kind, + RuntimeRequestKind::Apply | RuntimeRequestKind::Stop + ) { return Err(RuntimeError::Protocol( - "remove request cannot complete with an observation".into(), + "request kind cannot complete with an observation".into(), )); } - receipt.complete_with_observation(observation); + ensure_receipt_target(&receipt, &observation.unit_id, observation.generation)?; + if receipt.state == RuntimeRequestState::Completed { + if receipt.observation.as_ref() != Some(&observation) { + return Err(RuntimeError::Protocol( + "completed Runtime request result changed".into(), + )); + } + } else { + receipt.complete_with_observation(observation.clone()); + receipt.validate().map_err(RuntimeError::Protocol)?; + atomic_write(&receipt_path, &receipt, "request receipt")?; + #[cfg(test)] + test_failpoint("state.complete-observation.after-receipt-publish"); + } } + + record.observation = observation; record.validate().map_err(RuntimeError::Protocol)?; - atomic_write(&path, &record)?; + atomic_write(&record_path, &record, "state record")?; Ok(record) } fn complete_removal_sync(&self, removal: RuntimeRemoval) -> RuntimeResult { removal.validate().map_err(RuntimeError::Protocol)?; let _lock = self.lock(&removal.unit_id)?; - let path = self.record_path(&removal.unit_id)?; - if !path.is_file() { - return Err(RuntimeError::NotFound { - unit_id: removal.unit_id, - }); - } - let mut record = read_record(&path)?; + let record_path = self.record_path(&removal.unit_id, false)?; + let mut record = read_required_record(&record_path, &removal.unit_id)?; if removal.generation != record.spec.generation { return Err(RuntimeError::Protocol( "Runtime removal generation does not match stored unit".into(), )); } - let receipt = record - .receipt_mut(&removal.request_id) - .map_err(RuntimeError::Protocol)?; + let receipt_path = self.request_path(&removal.unit_id, &removal.request_id, false)?; + let mut receipt = + read_required_receipt(&receipt_path, &removal.unit_id, &removal.request_id)?; if receipt.kind != RuntimeRequestKind::Remove { return Err(RuntimeError::Protocol( "non-remove request completed with removal result".into(), )); } - receipt.complete_with_removal(removal.clone()); + ensure_receipt_target(&receipt, &removal.unit_id, removal.generation)?; + if receipt.state == RuntimeRequestState::Completed { + if receipt.removal.as_ref() != Some(&removal) { + return Err(RuntimeError::Protocol( + "completed Runtime removal result changed".into(), + )); + } + } else { + receipt.complete_with_removal(removal.clone()); + receipt.validate().map_err(RuntimeError::Protocol)?; + atomic_write(&receipt_path, &receipt, "request receipt")?; + #[cfg(test)] + test_failpoint("state.complete-removal.after-receipt-publish"); + } record.removed_at_ms = Some(removal.removed_at_ms); record.validate().map_err(RuntimeError::Protocol)?; - atomic_write(&path, &record)?; + atomic_write(&record_path, &record, "state record")?; + Ok(record) + } + + fn complete_exec_sync(&self, result: RuntimeExecResult) -> RuntimeResult { + result.validate().map_err(RuntimeError::Protocol)?; + let unit_id = result.observation.unit_id.clone(); + let _lock = self.lock(&unit_id)?; + let record_path = self.record_path(&unit_id, false)?; + let mut record = read_required_record(&record_path, &unit_id)?; + if record.removed_at_ms.is_some() { + return Err(RuntimeError::Protocol( + "cannot complete exec for an explicitly removed Runtime unit".into(), + )); + } + result + .observation + .validate_against(&record.spec) + .map_err(RuntimeError::Protocol)?; + validate_transition(&record.observation, &result.observation, &record.spec)?; + let receipt_path = self.request_path(&unit_id, &result.request_id, false)?; + let mut receipt = read_required_receipt(&receipt_path, &unit_id, &result.request_id)?; + if receipt.kind != RuntimeRequestKind::Exec { + return Err(RuntimeError::Protocol( + "non-exec request completed with exec result".into(), + )); + } + if receipt.state == RuntimeRequestState::Completed { + if receipt.exec_result.as_ref() != Some(&result) { + return Err(RuntimeError::Protocol( + "completed Runtime exec result changed".into(), + )); + } + } else { + receipt.complete_with_exec_result(result.clone()); + receipt.validate().map_err(RuntimeError::Protocol)?; + atomic_write(&receipt_path, &receipt, "request receipt")?; + #[cfg(test)] + test_failpoint("state.complete-exec.after-receipt-publish"); + } + record.observation = result.observation; + record.validate().map_err(RuntimeError::Protocol)?; + atomic_write(&record_path, &record, "state record")?; Ok(record) } @@ -243,22 +427,73 @@ impl FileRuntimeStateStore { ensure_directory(&self.root)?; let locks = self.root.join("locks"); ensure_directory(&locks)?; - let file = owner_only_open(&locks.join(format!("{}.lock", storage_key(unit_id))))?; + let file = owner_only_open( + &locks.join(format!("{}.lock", storage_key(unit_id))), + "Runtime state lock", + )?; file.lock_exclusive() .map_err(io_error("lock Runtime unit"))?; Ok(StateLock(file)) } - fn record_path(&self, unit_id: &str) -> RuntimeResult { + fn unit_directory(&self, unit_id: &str, create: bool) -> RuntimeResult { validate_unit_id(unit_id)?; - let records = self.root.join("units"); - ensure_directory(&records)?; - Ok(records.join(format!("{}.json", storage_key(unit_id)))) + let units = self.root.join("units"); + if create { + ensure_directory(&self.root)?; + ensure_directory(&units)?; + } else if path_exists(&units)? { + ensure_directory(&units)?; + } + let key = storage_key(unit_id); + let legacy = units.join(format!("{key}.json")); + if path_exists(&legacy)? { + return Err(RuntimeError::Protocol(format!( + "legacy Runtime unit record {} requires explicit migration", + legacy.display() + ))); + } + let unit = units.join(key); + if create || path_exists(&unit)? { + ensure_directory(&unit)?; + } + Ok(unit) + } + + fn record_path(&self, unit_id: &str, create: bool) -> RuntimeResult { + Ok(self.unit_directory(unit_id, create)?.join("record.json")) + } + + fn request_path( + &self, + unit_id: &str, + request_id: &str, + create: bool, + ) -> RuntimeResult { + validate_request_id(request_id)?; + let requests = self.unit_directory(unit_id, create)?.join("requests"); + if create || path_exists(&requests)? { + ensure_directory(&requests)?; + } + Ok(requests.join(format!("{}.json", storage_key(request_id)))) } } #[async_trait] impl RuntimeStateStore for FileRuntimeStateStore { + async fn acquire_operation_lease( + &self, + unit_id: &str, + ) -> RuntimeResult> { + let store = self.clone(); + let unit_id = unit_id.to_owned(); + let lease = + tokio::task::spawn_blocking(move || store.acquire_operation_lease_sync(&unit_id)) + .await + .map_err(task_error)??; + Ok(Box::new(lease)) + } + async fn reserve_apply( &self, request: &RuntimeApplyRequest, @@ -284,6 +519,18 @@ impl RuntimeStateStore for FileRuntimeStateStore { .map_err(task_error)? } + async fn reserve_exec( + &self, + request: &RuntimeExecRequest, + now_ms: u64, + ) -> RuntimeResult { + let store = self.clone(); + let request = request.clone(); + tokio::task::spawn_blocking(move || store.reserve_exec_sync(request, now_ms)) + .await + .map_err(task_error)? + } + async fn load(&self, unit_id: &str) -> RuntimeResult { let store = self.clone(); let unit_id = unit_id.to_owned(); @@ -292,6 +539,19 @@ impl RuntimeStateStore for FileRuntimeStateStore { .map_err(task_error)? } + async fn load_request( + &self, + unit_id: &str, + request_id: &str, + ) -> RuntimeResult { + let store = self.clone(); + let unit_id = unit_id.to_owned(); + let request_id = request_id.to_owned(); + tokio::task::spawn_blocking(move || store.load_request_sync(&unit_id, &request_id)) + .await + .map_err(task_error)? + } + async fn update_observation( &self, request_id: Option<&str>, @@ -312,21 +572,25 @@ impl RuntimeStateStore for FileRuntimeStateStore { .await .map_err(task_error)? } + + async fn complete_exec(&self, result: &RuntimeExecResult) -> RuntimeResult { + let store = self.clone(); + let result = result.clone(); + tokio::task::spawn_blocking(move || store.complete_exec_sync(result)) + .await + .map_err(task_error)? + } } fn reservation( record: RuntimeUnitRecord, - request_id: &str, -) -> RuntimeResult { - let receipt = record - .receipt(request_id) - .cloned() - .ok_or_else(|| RuntimeError::Protocol("reserved request receipt is missing".into()))?; - Ok(RuntimeStateReservation { + receipt: RuntimeRequestReceipt, +) -> RuntimeStateReservation { + RuntimeStateReservation { dispatch: receipt.state == RuntimeRequestState::Pending, record, receipt, - }) + } } fn ensure_same_request( @@ -342,6 +606,104 @@ fn ensure_same_request( Ok(()) } +fn ensure_receipt_target( + receipt: &RuntimeRequestReceipt, + unit_id: &str, + generation: u64, +) -> RuntimeResult<()> { + if receipt.unit_id != unit_id || receipt.generation != generation { + return Err(RuntimeError::RequestConflict { + request_id: receipt.request_id.clone(), + }); + } + Ok(()) +} + +fn ensure_current_generation( + record: &RuntimeUnitRecord, + unit_id: &str, + requested: u64, +) -> RuntimeResult<()> { + if requested < record.spec.generation { + return Err(RuntimeError::StaleGeneration { + unit_id: unit_id.into(), + requested, + current: record.spec.generation, + }); + } + if requested != record.spec.generation { + return Err(RuntimeError::GenerationConflict { + unit_id: unit_id.into(), + generation: requested, + }); + } + Ok(()) +} + +fn reconcile_completed_observation( + record: &mut RuntimeUnitRecord, + receipt: &RuntimeRequestReceipt, +) -> RuntimeResult { + let Some(observation) = &receipt.observation else { + return Ok(false); + }; + if record.removed_at_ms.is_some() + || record.spec.generation != receipt.generation + || record.spec.unit_id != receipt.unit_id + || record.observation == *observation + || record.observation.observed_at_ms > observation.observed_at_ms + { + return Ok(false); + } + if validate_transition(&record.observation, observation, &record.spec).is_ok() { + record.observation = observation.clone(); + record.validate().map_err(RuntimeError::Protocol)?; + return Ok(true); + } + Ok(false) +} + +fn reconcile_completed_removal( + record: &mut RuntimeUnitRecord, + receipt: &RuntimeRequestReceipt, +) -> RuntimeResult { + let Some(removal) = &receipt.removal else { + return Ok(false); + }; + if record.spec.unit_id == receipt.unit_id + && record.spec.generation == receipt.generation + && record.removed_at_ms.is_none() + { + record.removed_at_ms = Some(removal.removed_at_ms); + record.validate().map_err(RuntimeError::Protocol)?; + return Ok(true); + } + Ok(false) +} + +fn reconcile_completed_exec( + record: &mut RuntimeUnitRecord, + receipt: &RuntimeRequestReceipt, +) -> RuntimeResult { + let Some(result) = &receipt.exec_result else { + return Ok(false); + }; + if record.removed_at_ms.is_some() + || record.spec.generation != receipt.generation + || record.spec.unit_id != receipt.unit_id + || record.observation == result.observation + || record.observation.observed_at_ms > result.observation.observed_at_ms + { + return Ok(false); + } + if validate_transition(&record.observation, &result.observation, &record.spec).is_ok() { + record.observation = result.observation.clone(); + record.validate().map_err(RuntimeError::Protocol)?; + return Ok(true); + } + Ok(false) +} + pub(crate) fn validate_transition( current: &RuntimeObservation, next: &RuntimeObservation, @@ -352,7 +714,8 @@ pub(crate) fn validate_transition( .map_err(RuntimeError::Protocol)?; next.validate_against(spec) .map_err(RuntimeError::Protocol)?; - if current.provider_resource_id.is_some() + if current.state != RuntimeUnitState::Unknown + && current.provider_resource_id.is_some() && current.provider_resource_id != next.provider_resource_id { return Err(RuntimeError::Protocol( @@ -373,7 +736,6 @@ pub(crate) fn validate_transition( return Ok(()); } let allowed = current.state == next.state - || current.state == RuntimeUnitState::Unknown || matches!( (current.state, next.state), (RuntimeUnitState::Accepted, RuntimeUnitState::Preparing) @@ -404,6 +766,13 @@ pub(crate) fn validate_transition( | (RuntimeUnitState::Stopping, RuntimeUnitState::Stopped) | (RuntimeUnitState::Stopping, RuntimeUnitState::Failed) | (RuntimeUnitState::Stopping, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Preparing) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Starting) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Running) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Stopping) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Failed) ); if !allowed { return Err(RuntimeError::Protocol(format!( @@ -422,125 +791,32 @@ impl Drop for StateLock { } } -fn storage_key(unit_id: &str) -> String { - format!("{:x}", Sha256::digest(unit_id.as_bytes())) -} - -fn validate_unit_id(unit_id: &str) -> RuntimeResult<()> { - crate::contract::validate_id("unit_id", unit_id, 512).map_err(RuntimeError::InvalidRequest) -} +struct FileOperationLease(File); -fn ensure_directory(path: &Path) -> RuntimeResult<()> { - if path.exists() { - let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state path"))?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(RuntimeError::Protocol(format!( - "Runtime state path {} is not a real directory", - path.display() - ))); - } - } else { - std::fs::create_dir_all(path).map_err(io_error("create state directory"))?; - } - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) - .map_err(io_error("secure state directory"))?; - } - Ok(()) -} +impl RuntimeOperationLease for FileOperationLease {} -fn owner_only_open(path: &Path) -> RuntimeResult { - reject_symlink(path, "Runtime state lock")?; - let mut options = OpenOptions::new(); - options.create(true).read(true).write(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600).custom_flags(libc::O_NOFOLLOW); - } - let file = options.open(path).map_err(io_error("open state lock"))?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - file.set_permissions(std::fs::Permissions::from_mode(0o600)) - .map_err(io_error("secure state lock"))?; +impl Drop for FileOperationLease { + fn drop(&mut self) { + let _ = FileExt::unlock(&self.0); } - Ok(file) } -fn read_record(path: &Path) -> RuntimeResult { - let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state record"))?; - if metadata.file_type().is_symlink() || !metadata.is_file() { - return Err(RuntimeError::Protocol( - "Runtime state record is not a regular file".into(), - )); - } - let mut bytes = Vec::new(); - let mut options = OpenOptions::new(); - options.read(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.custom_flags(libc::O_NOFOLLOW); - } - options - .open(path) - .and_then(|mut file| file.read_to_end(&mut bytes)) - .map_err(io_error("read state record"))?; - let record: RuntimeUnitRecord = serde_json::from_slice(&bytes) - .map_err(|error| RuntimeError::Protocol(format!("invalid state record: {error}")))?; - record.validate().map_err(RuntimeError::Protocol)?; - Ok(record) +fn storage_key(value: &str) -> String { + format!("{:x}", Sha256::digest(value.as_bytes())) } -fn reject_symlink(path: &Path, label: &str) -> RuntimeResult<()> { - match std::fs::symlink_metadata(path) { - Ok(metadata) if metadata.file_type().is_symlink() => Err(RuntimeError::Protocol(format!( - "{label} {} must not be a symbolic link", - path.display() - ))), - Ok(_) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(io_error("inspect state file")(error)), - } +fn validate_unit_id(unit_id: &str) -> RuntimeResult<()> { + crate::contract::validate_id("unit_id", unit_id, 512).map_err(RuntimeError::InvalidRequest) } -fn atomic_write(path: &Path, record: &RuntimeUnitRecord) -> RuntimeResult<()> { - let parent = path - .parent() - .ok_or_else(|| RuntimeError::Protocol("state record has no parent".into()))?; - let bytes = serde_json::to_vec(record) - .map_err(|error| RuntimeError::Protocol(format!("encode state record: {error}")))?; - let mut temporary = - tempfile::NamedTempFile::new_in(parent).map_err(io_error("create state staging file"))?; - temporary - .write_all(&bytes) - .and_then(|()| temporary.as_file().sync_all()) - .map_err(io_error("write state record"))?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - temporary - .as_file() - .set_permissions(std::fs::Permissions::from_mode(0o600)) - .map_err(io_error("secure state record"))?; - } - temporary - .persist(path) - .map_err(|error| io_error("publish state record")(error.error))?; - #[cfg(unix)] - File::open(parent) - .and_then(|directory| directory.sync_all()) - .map_err(io_error("sync state directory"))?; - Ok(()) +fn validate_request_id(request_id: &str) -> RuntimeResult<()> { + crate::contract::validate_id("request_id", request_id, 512) + .map_err(RuntimeError::InvalidRequest) } fn task_error(error: tokio::task::JoinError) -> RuntimeError { RuntimeError::Transport(format!("Runtime state task failed: {error}")) } -fn io_error(action: &'static str) -> impl FnOnce(std::io::Error) -> RuntimeError { - move |error| RuntimeError::Transport(format!("could not {action}: {error}")) -} +#[cfg(test)] +mod tests; diff --git a/src/state/file/fs.rs b/src/state/file/fs.rs new file mode 100644 index 0000000..a4a54d7 --- /dev/null +++ b/src/state/file/fs.rs @@ -0,0 +1,317 @@ +use crate::state::{RuntimeRequestReceipt, RuntimeUnitRecord}; +use crate::{RuntimeError, RuntimeResult}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::Path; + +const MAX_RECORD_BYTES: u64 = 8 * 1024 * 1024; +const MAX_RECEIPT_BYTES: u64 = 40 * 1024 * 1024; + +pub(super) fn ensure_directory(path: &Path) -> RuntimeResult<()> { + if path_exists(path)? { + let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state path"))?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(RuntimeError::Protocol(format!( + "Runtime state path {} is not a real directory", + path.display() + ))); + } + verify_owner(&metadata, path, "state directory")?; + } else { + std::fs::create_dir_all(path).map_err(io_error("create state directory"))?; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) + .map_err(io_error("secure state directory"))?; + } + Ok(()) +} + +pub(super) fn owner_only_open(path: &Path, label: &str) -> RuntimeResult { + reject_symlink(path, label)?; + let mut options = OpenOptions::new(); + options.create(true).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let file = options.open(path).map_err(io_error("open state lock"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .map_err(io_error("secure state lock"))?; + } + let metadata = file.metadata().map_err(io_error("inspect state lock"))?; + verify_owner_only_file(&metadata, path, label)?; + Ok(file) +} + +pub(super) fn read_required_record(path: &Path, unit_id: &str) -> RuntimeResult { + let record = read_optional_record(path)?.ok_or_else(|| RuntimeError::NotFound { + unit_id: unit_id.into(), + })?; + if record.spec.unit_id != unit_id { + return Err(RuntimeError::Protocol("Runtime state key mismatch".into())); + } + Ok(record) +} + +pub(super) fn read_optional_record(path: &Path) -> RuntimeResult> { + read_optional_json(path, MAX_RECORD_BYTES, "state record") +} + +pub(super) fn read_required_receipt( + path: &Path, + unit_id: &str, + request_id: &str, +) -> RuntimeResult { + let receipt = read_optional_receipt(path)?.ok_or_else(|| RuntimeError::RequestNotFound { + unit_id: unit_id.into(), + request_id: request_id.into(), + })?; + if receipt.unit_id != unit_id || receipt.request_id != request_id { + return Err(RuntimeError::Protocol( + "Runtime request receipt storage key mismatch".into(), + )); + } + Ok(receipt) +} + +pub(super) fn read_optional_receipt(path: &Path) -> RuntimeResult> { + let receipt: Option = + read_optional_json(path, MAX_RECEIPT_BYTES, "request receipt")?; + if let Some(receipt) = &receipt { + receipt.validate().map_err(RuntimeError::Protocol)?; + } + Ok(receipt) +} + +fn read_optional_json( + path: &Path, + max_bytes: u64, + label: &str, +) -> RuntimeResult> { + if !regular_file_exists(path, label)? { + return Ok(None); + } + let metadata = std::fs::symlink_metadata(path).map_err(io_error("inspect state file"))?; + verify_owner_only_file(&metadata, path, label)?; + if metadata.len() > max_bytes { + return Err(RuntimeError::Protocol(format!( + "Runtime {label} exceeds its size limit" + ))); + } + let capacity = usize::try_from(metadata.len()).map_err(|_| { + RuntimeError::Protocol(format!("Runtime {label} size cannot be represented")) + })?; + let mut bytes = Vec::with_capacity(capacity); + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + options + .open(path) + .and_then(|mut file| file.read_to_end(&mut bytes)) + .map_err(io_error("read state file"))?; + let value = serde_json::from_slice(&bytes) + .map_err(|error| RuntimeError::Protocol(format!("invalid {label}: {error}")))?; + Ok(Some(value)) +} + +fn regular_file_exists(path: &Path, label: &str) -> RuntimeResult { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + Err(RuntimeError::Protocol(format!( + "Runtime {label} {} is not a regular file", + path.display() + ))) + } + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(io_error("inspect state file")(error)), + } +} + +pub(super) fn path_exists(path: &Path) -> RuntimeResult { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(io_error("inspect state path")(error)), + } +} + +fn reject_symlink(path: &Path, label: &str) -> RuntimeResult<()> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => Err(RuntimeError::Protocol(format!( + "{label} {} must not be a symbolic link", + path.display() + ))), + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(io_error("inspect state file")(error)), + } +} + +#[cfg(unix)] +fn verify_owner(metadata: &std::fs::Metadata, path: &Path, label: &str) -> RuntimeResult<()> { + use std::os::unix::fs::MetadataExt; + if metadata.uid() != unsafe { libc::geteuid() } { + return Err(RuntimeError::Protocol(format!( + "Runtime {label} {} is owned by another user", + path.display() + ))); + } + Ok(()) +} + +#[cfg(not(unix))] +fn verify_owner(_metadata: &std::fs::Metadata, _path: &Path, _label: &str) -> RuntimeResult<()> { + Ok(()) +} + +fn verify_owner_only_file( + metadata: &std::fs::Metadata, + path: &Path, + label: &str, +) -> RuntimeResult<()> { + verify_owner(metadata, path, label)?; + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + if metadata.permissions().mode() & 0o077 != 0 || metadata.nlink() != 1 { + return Err(RuntimeError::Protocol(format!( + "Runtime {label} {} is not an owner-only unlinked file", + path.display() + ))); + } + } + Ok(()) +} + +pub(super) fn atomic_write(path: &Path, value: &T, label: &str) -> RuntimeResult<()> { + let parent = path + .parent() + .ok_or_else(|| RuntimeError::Protocol(format!("{label} has no parent")))?; + let bytes = serde_json::to_vec(value) + .map_err(|error| RuntimeError::Protocol(format!("encode {label}: {error}")))?; + + cleanup_stale_staging_files(parent)?; + #[cfg(test)] + super::tests::inject_io_fault("state.atomic-write.before-create") + .map_err(io_error("create state staging file"))?; + + let target_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| RuntimeError::Protocol(format!("{label} has an invalid file name")))?; + let staging_prefix = format!(".a3s-runtime-{target_name}-"); + let mut temporary = tempfile::Builder::new() + .prefix(&staging_prefix) + .suffix(".tmp") + .tempfile_in(parent) + .map_err(io_error("create state staging file"))?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-create"); + + let split = bytes.len().div_ceil(2); + temporary + .write_all(&bytes[..split]) + .map_err(io_error("write state file"))?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-partial-write"); + temporary + .write_all(&bytes[split..]) + .map_err(io_error("write state file"))?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-complete-write"); + temporary + .as_file() + .sync_all() + .map_err(io_error("sync state file"))?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-file-sync"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + temporary + .as_file() + .set_permissions(std::fs::Permissions::from_mode(0o600)) + .map_err(io_error("secure state file"))?; + } + #[cfg(test)] + test_failpoint("state.atomic-write.after-permissions"); + temporary + .as_file() + .sync_all() + .map_err(io_error("sync secured state file"))?; + temporary + .persist(path) + .map_err(|error| io_error("publish state file")(error.error))?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-publish"); + sync_directory(parent)?; + #[cfg(test)] + test_failpoint("state.atomic-write.after-directory-sync"); + Ok(()) +} + +fn cleanup_stale_staging_files(parent: &Path) -> RuntimeResult<()> { + let mut removed = false; + for entry in std::fs::read_dir(parent).map_err(io_error("scan state staging files"))? { + let entry = entry.map_err(io_error("scan state staging file"))?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !(name.starts_with(".tmp") + || name.starts_with(".a3s-runtime-") && name.ends_with(".tmp")) + { + continue; + } + + let path = entry.path(); + let metadata = + std::fs::symlink_metadata(&path).map_err(io_error("inspect state staging file"))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(RuntimeError::Protocol(format!( + "Runtime state staging file {} is not a regular file", + path.display() + ))); + } + verify_owner_only_file(&metadata, &path, "state staging file")?; + std::fs::remove_file(&path).map_err(io_error("remove stale state staging file"))?; + removed = true; + } + if removed { + sync_directory(parent)?; + } + Ok(()) +} + +#[cfg(unix)] +fn sync_directory(path: &Path) -> RuntimeResult<()> { + File::open(path) + .and_then(|directory| directory.sync_all()) + .map_err(io_error("sync state directory")) +} + +#[cfg(not(unix))] +fn sync_directory(_path: &Path) -> RuntimeResult<()> { + Ok(()) +} + +pub(super) fn io_error(action: &'static str) -> impl FnOnce(std::io::Error) -> RuntimeError { + move |error| RuntimeError::Transport(format!("could not {action}: {error}")) +} + +#[cfg(test)] +pub(super) fn test_failpoint(name: &str) { + super::tests::hit_failpoint(name); +} diff --git a/src/state/file/tests.rs b/src/state/file/tests.rs new file mode 100644 index 0000000..e3ccda4 --- /dev/null +++ b/src/state/file/tests.rs @@ -0,0 +1,716 @@ +use super::*; +use crate::contract::{ + ArtifactRef, IsolationLevel, NetworkMode, ResourceLimits, RestartPolicy, RuntimeActionRequest, + RuntimeApplyRequest, RuntimeExecRequest, RuntimeExecResult, RuntimeNetworkSpec, + RuntimeObservation, RuntimeProcessSpec, RuntimeRemoval, RuntimeUnitClass, RuntimeUnitSpec, + RuntimeUnitState, +}; +use crate::RuntimeRequestState; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; +use std::thread; +use std::time::{Duration, Instant}; + +const NOW: u64 = 10_000; +const UNIT_ID: &str = "receipt-first-unit"; +const APPLY_ID: &str = "receipt-first-apply"; +const STOP_ID: &str = "receipt-first-stop"; +const REMOVE_ID: &str = "receipt-first-remove"; +const EXEC_ID: &str = "receipt-first-exec"; +const FAILPOINT_ENV: &str = "A3S_RUNTIME_TEST_FAILPOINT"; +const IO_FAULT_ENV: &str = "A3S_RUNTIME_TEST_IO_FAULT"; +const READY_ENV: &str = "A3S_RUNTIME_TEST_FAILPOINT_READY"; +const ROOT_ENV: &str = "A3S_RUNTIME_TEST_STATE_ROOT"; +const OPERATION_ENV: &str = "A3S_RUNTIME_TEST_COMPLETION_KIND"; +const OBSERVATION_FAILPOINT: &str = "state.complete-observation.after-receipt-publish"; +const REMOVAL_FAILPOINT: &str = "state.complete-removal.after-receipt-publish"; +const EXEC_FAILPOINT: &str = "state.complete-exec.after-receipt-publish"; + +pub(super) fn hit_failpoint(name: &str) { + if !matches!(std::env::var(FAILPOINT_ENV), Ok(value) if value == name) { + return; + } + let ready = std::env::var(READY_ENV).expect("test failpoint ready path"); + std::fs::write(ready, name).expect("publish test failpoint readiness"); + loop { + thread::park_timeout(Duration::from_secs(60)); + } +} + +pub(super) fn inject_io_fault(point: &str) -> std::io::Result<()> { + let Ok(fault) = std::env::var(IO_FAULT_ENV) else { + return Ok(()); + }; + let Some(kind) = fault + .strip_prefix(point) + .and_then(|suffix| suffix.strip_prefix('.')) + else { + return Ok(()); + }; + + #[cfg(unix)] + { + let raw_error = match kind { + "enospc" | "inode-exhaustion" => libc::ENOSPC, + "read-only" => libc::EROFS, + _ => return Ok(()), + }; + Err(std::io::Error::from_raw_os_error(raw_error)) + } + + #[cfg(not(unix))] + { + match kind { + "enospc" | "inode-exhaustion" | "read-only" => Err(std::io::Error::other(format!( + "injected Runtime state I/O fault: {kind}" + ))), + _ => Ok(()), + } + } +} + +fn digest(character: char) -> String { + format!("sha256:{}", character.to_string().repeat(64)) +} + +fn spec() -> RuntimeUnitSpec { + RuntimeUnitSpec { + schema: RuntimeUnitSpec::SCHEMA.into(), + unit_id: UNIT_ID.into(), + generation: 1, + class: RuntimeUnitClass::Service, + artifact: ArtifactRef { + uri: format!( + "oci://registry.example/a3s/runtime@sha256:{}", + "a".repeat(64) + ), + digest: digest('a'), + media_type: "application/vnd.oci.image.manifest.v1+json".into(), + }, + process: RuntimeProcessSpec { + command: vec!["/bin/service".into()], + args: Vec::new(), + working_directory: None, + environment: BTreeMap::new(), + }, + mounts: Vec::new(), + secrets: Vec::new(), + network: RuntimeNetworkSpec { + mode: NetworkMode::None, + ports: Vec::new(), + }, + resources: ResourceLimits { + cpu_millis: 100, + memory_bytes: 64 * 1024 * 1024, + pids: 32, + ephemeral_storage_bytes: None, + execution_timeout_ms: None, + }, + isolation: IsolationLevel::Container, + health: None, + restart: RestartPolicy::Always, + outputs: Vec::new(), + semantics_profile_digest: None, + } +} + +fn apply_request() -> RuntimeApplyRequest { + RuntimeApplyRequest { + schema: RuntimeApplyRequest::SCHEMA.into(), + request_id: APPLY_ID.into(), + deadline_at_ms: Some(NOW + 60_000), + spec: spec(), + } +} + +fn action(request_id: &str) -> RuntimeActionRequest { + RuntimeActionRequest { + schema: RuntimeActionRequest::SCHEMA.into(), + request_id: request_id.into(), + unit_id: UNIT_ID.into(), + generation: 1, + deadline_at_ms: Some(NOW + 60_000), + } +} + +fn exec_request() -> RuntimeExecRequest { + RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: EXEC_ID.into(), + unit_id: UNIT_ID.into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: Some(NOW + 60_000), + } +} + +fn provider_observation( + mut observation: RuntimeObservation, + state: RuntimeUnitState, + observed_at_ms: u64, +) -> RuntimeObservation { + observation.state = state; + observation.provider_resource_id = Some(format!("provider/{UNIT_ID}/1")); + observation.provider_build = Some("receipt-first-driver/1".into()); + observation.observed_at_ms = observed_at_ms; + observation.started_at_ms = Some(NOW); + observation.finished_at_ms = state.is_terminal().then_some(observed_at_ms); + observation.health = None; + observation.outputs.clear(); + observation.failure = None; + observation +} + +fn prepare_running(store: &FileRuntimeStateStore) -> RuntimeObservation { + let request = apply_request(); + let reservation = store + .reserve_apply_sync(request.clone(), NOW) + .expect("reserve initial apply"); + let running = provider_observation( + reservation.record.observation, + RuntimeUnitState::Running, + NOW + 1, + ); + store + .update_observation_sync(Some(request.request_id), running.clone()) + .expect("complete initial apply"); + running +} + +fn spawn_completion_helper(root: &Path, ready: &Path, operation: &str, failpoint: &str) -> Child { + Command::new(std::env::current_exe().expect("current test executable")) + .arg("subprocess_receipt_first_completion_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env(ROOT_ENV, root) + .env(READY_ENV, ready) + .env(OPERATION_ENV, operation) + .env(FAILPOINT_ENV, failpoint) + .spawn() + .expect("spawn receipt-first completion helper") +} + +fn kill_at_failpoint(child: &mut Child, ready: &Path, case_id: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + while !ready.is_file() { + if let Some(status) = child.try_wait().expect("inspect completion helper") { + panic!("{case_id}: helper exited before failpoint: {status}"); + } + assert!( + Instant::now() < deadline, + "{case_id}: helper did not reach failpoint" + ); + thread::sleep(Duration::from_millis(10)); + } + child.kill().expect("kill completion helper"); + let status = child.wait().expect("wait for killed completion helper"); + assert!(!status.success(), "{case_id}: helper was not killed"); +} + +fn run_receipt_first_crash( + operation: &str, + failpoint: &str, + prepare: impl FnOnce(&FileRuntimeStateStore), + verify: impl FnOnce(&FileRuntimeStateStore), +) { + let directory = tempfile::tempdir().expect("receipt-first state root"); + let store = FileRuntimeStateStore::new(directory.path()); + prepare(&store); + let ready = directory.path().join(format!("{operation}.ready")); + let mut child = spawn_completion_helper(directory.path(), &ready, operation, failpoint); + kill_at_failpoint(&mut child, &ready, operation); + verify(&FileRuntimeStateStore::new(directory.path())); +} + +#[test] +fn subprocess_receipt_first_completion_helper() { + let Ok(root) = std::env::var(ROOT_ENV) else { + return; + }; + let operation = std::env::var(OPERATION_ENV).expect("completion kind"); + let store = FileRuntimeStateStore::new(PathBuf::from(root)); + let record = store.load_sync(UNIT_ID).expect("load helper unit"); + match operation.as_str() { + "crash-apply-receipt-first" => { + let running = + provider_observation(record.observation, RuntimeUnitState::Running, NOW + 1); + store + .update_observation_sync(Some(APPLY_ID.into()), running) + .expect("complete apply"); + } + "crash-stop-receipt-first" => { + let stopped = + provider_observation(record.observation, RuntimeUnitState::Stopped, NOW + 2); + store + .update_observation_sync(Some(STOP_ID.into()), stopped) + .expect("complete stop"); + } + "crash-remove-receipt-first" => { + store + .complete_removal_sync(RuntimeRemoval { + schema: RuntimeRemoval::SCHEMA.into(), + request_id: REMOVE_ID.into(), + unit_id: UNIT_ID.into(), + generation: 1, + removed_at_ms: NOW + 3, + already_absent: false, + }) + .expect("complete removal"); + } + "crash-exec-receipt-first" => { + let mut refreshed = record.observation; + refreshed.observed_at_ms += 1; + store + .complete_exec_sync(RuntimeExecResult { + schema: RuntimeExecResult::SCHEMA.into(), + request_id: EXEC_ID.into(), + observation: refreshed, + exit_code: 0, + stdout: "ok\n".into(), + stderr: String::new(), + truncated: false, + }) + .expect("complete exec"); + } + other => panic!("unknown receipt-first completion kind {other:?}"), + } +} + +#[test] +fn state_crash_001_receipt_first_process_kills_reconcile_every_result_kind() { + run_receipt_first_crash( + "crash-apply-receipt-first", + OBSERVATION_FAILPOINT, + |store| { + store + .reserve_apply_sync(apply_request(), NOW) + .expect("reserve apply"); + }, + |store| { + assert_eq!( + store + .load_sync(UNIT_ID) + .expect("pre-replay unit") + .observation + .state, + RuntimeUnitState::Accepted + ); + assert_eq!( + store + .load_request_sync(UNIT_ID, APPLY_ID) + .expect("completed apply receipt") + .state, + RuntimeRequestState::Completed + ); + let replay = store + .reserve_apply_sync(apply_request(), NOW + 2) + .expect("replay apply"); + assert!(!replay.dispatch); + assert_eq!(replay.record.observation.state, RuntimeUnitState::Running); + assert_eq!( + store + .load_sync(UNIT_ID) + .expect("reconciled unit") + .observation, + replay.record.observation + ); + }, + ); + + run_receipt_first_crash( + "crash-stop-receipt-first", + OBSERVATION_FAILPOINT, + |store| { + prepare_running(store); + store + .reserve_action_sync(RuntimeActionKind::Stop, action(STOP_ID), NOW + 1) + .expect("reserve stop"); + }, + |store| { + assert_eq!( + store + .load_sync(UNIT_ID) + .expect("pre-replay unit") + .observation + .state, + RuntimeUnitState::Running + ); + let replay = store + .reserve_action_sync(RuntimeActionKind::Stop, action(STOP_ID), NOW + 3) + .expect("replay stop"); + assert!(!replay.dispatch); + assert_eq!(replay.record.observation.state, RuntimeUnitState::Stopped); + }, + ); + + run_receipt_first_crash( + "crash-remove-receipt-first", + REMOVAL_FAILPOINT, + |store| { + prepare_running(store); + store + .reserve_action_sync(RuntimeActionKind::Remove, action(REMOVE_ID), NOW + 1) + .expect("reserve removal"); + }, + |store| { + assert!(store + .load_sync(UNIT_ID) + .expect("pre-replay unit") + .removed_at_ms + .is_none()); + let replay = store + .reserve_action_sync(RuntimeActionKind::Remove, action(REMOVE_ID), NOW + 4) + .expect("replay removal"); + assert!(!replay.dispatch); + assert_eq!(replay.record.removed_at_ms, Some(NOW + 3)); + }, + ); + + run_receipt_first_crash( + "crash-exec-receipt-first", + EXEC_FAILPOINT, + |store| { + prepare_running(store); + store + .reserve_exec_sync(exec_request(), NOW + 1) + .expect("reserve exec"); + }, + |store| { + let before = store + .load_sync(UNIT_ID) + .expect("pre-replay unit") + .observation + .observed_at_ms; + let replay = store + .reserve_exec_sync(exec_request(), NOW + 2) + .expect("replay exec"); + assert!(!replay.dispatch); + assert!(replay.record.observation.observed_at_ms > before); + assert_eq!( + replay.receipt.state, + RuntimeRequestState::Completed, + "exec result must remain durable after process kill" + ); + }, + ); +} + +#[cfg(unix)] +#[test] +fn state_fs_001_hard_linked_lock_record_and_receipt_files_fail_closed() { + let directory = tempfile::tempdir().expect("hard-link state root"); + let store = FileRuntimeStateStore::new(directory.path()); + store + .reserve_apply_sync(apply_request(), NOW) + .expect("reserve hard-link fixture"); + let key = storage_key(UNIT_ID); + + let state_lock = directory.path().join("locks").join(format!("{key}.lock")); + let state_lock_alias = directory.path().join("state-lock.alias"); + std::fs::hard_link(&state_lock, &state_lock_alias).expect("hard-link state lock"); + assert!(matches!( + store.load_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); + std::fs::remove_file(state_lock_alias).expect("remove state-lock alias"); + + let record = store.record_path(UNIT_ID, false).expect("record path"); + let record_alias = directory.path().join("record.alias"); + std::fs::hard_link(&record, &record_alias).expect("hard-link record"); + assert!(matches!( + store.load_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); + std::fs::remove_file(record_alias).expect("remove record alias"); + + let receipt = store + .request_path(UNIT_ID, APPLY_ID, false) + .expect("receipt path"); + let receipt_alias = directory.path().join("receipt.alias"); + std::fs::hard_link(&receipt, &receipt_alias).expect("hard-link receipt"); + assert!(matches!( + store.load_request_sync(UNIT_ID, APPLY_ID), + Err(RuntimeError::Protocol(_)) + )); + std::fs::remove_file(receipt_alias).expect("remove receipt alias"); + + drop( + store + .acquire_operation_lease_sync(UNIT_ID) + .expect("create operation lock"), + ); + let operation_lock = directory + .path() + .join("operations") + .join(format!("{key}.lock")); + let operation_alias = directory.path().join("operation-lock.alias"); + std::fs::hard_link(&operation_lock, &operation_alias).expect("hard-link operation lock"); + assert!(matches!( + store.acquire_operation_lease_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); +} + +#[cfg(unix)] +#[test] +fn state_fs_002_corrupt_permission_and_non_regular_boundaries_fail_closed() { + use std::os::unix::fs::PermissionsExt; + use std::os::unix::net::UnixListener; + + let truncated = tempfile::tempdir().expect("truncated state root"); + let store = FileRuntimeStateStore::new(truncated.path()); + store + .reserve_apply_sync(apply_request(), NOW) + .expect("reserve truncated fixture"); + let record = store.record_path(UNIT_ID, false).expect("record path"); + std::fs::write(&record, b"{").expect("truncate record"); + assert!(matches!( + store.load_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); + + let invalid_utf8 = tempfile::tempdir().expect("invalid UTF-8 state root"); + let store = FileRuntimeStateStore::new(invalid_utf8.path()); + store + .reserve_apply_sync(apply_request(), NOW) + .expect("reserve invalid UTF-8 fixture"); + let receipt = store + .request_path(UNIT_ID, APPLY_ID, false) + .expect("receipt path"); + std::fs::write(&receipt, [0xff]).expect("corrupt receipt encoding"); + assert!(matches!( + store.load_request_sync(UNIT_ID, APPLY_ID), + Err(RuntimeError::Protocol(_)) + )); + + let permissions = tempfile::tempdir().expect("permission state root"); + let store = FileRuntimeStateStore::new(permissions.path()); + store + .reserve_apply_sync(apply_request(), NOW) + .expect("reserve permission fixture"); + let record = store.record_path(UNIT_ID, false).expect("record path"); + std::fs::set_permissions(&record, std::fs::Permissions::from_mode(0o644)) + .expect("relax record permissions"); + assert!(matches!( + store.load_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); + + let non_regular = tempfile::tempdir().expect("non-regular state root"); + let store = FileRuntimeStateStore::new(non_regular.path()); + let record = store.record_path(UNIT_ID, true).expect("record path"); + let _socket = UnixListener::bind(&record).expect("bind record socket"); + assert!(matches!( + store.load_sync(UNIT_ID), + Err(RuntimeError::Protocol(_)) + )); +} + +fn spawn_atomic_crash_helper(root: &Path, ready: &Path, failpoint: &str) -> Child { + Command::new(std::env::current_exe().expect("current test executable")) + .arg("subprocess_atomic_write_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env(ROOT_ENV, root) + .env(READY_ENV, ready) + .env(FAILPOINT_ENV, failpoint) + .spawn() + .expect("spawn atomic-write helper") +} + +fn spawn_io_fault_helper(root: &Path, result: &Path, request_id: &str, fault: &str) -> Child { + Command::new(std::env::current_exe().expect("current test executable")) + .arg("subprocess_atomic_io_fault_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env(ROOT_ENV, root) + .env(READY_ENV, result) + .env(OPERATION_ENV, request_id) + .env(IO_FAULT_ENV, fault) + .spawn() + .expect("spawn atomic I/O-fault helper") +} + +fn wait_for_success(child: &mut Child, case_id: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if let Some(status) = child.try_wait().expect("inspect state helper") { + assert!(status.success(), "{case_id}: helper failed: {status}"); + return; + } + if Instant::now() >= deadline { + child.kill().expect("kill timed-out state helper"); + child.wait().expect("wait for timed-out state helper"); + panic!("{case_id}: helper timed out"); + } + thread::sleep(Duration::from_millis(10)); + } +} + +fn staging_files(root: &Path) -> Vec { + let mut directories = vec![root.to_path_buf()]; + let mut staging = Vec::new(); + while let Some(directory) = directories.pop() { + for entry in std::fs::read_dir(directory).expect("scan state test directory") { + let entry = entry.expect("read state test entry"); + let path = entry.path(); + let file_type = entry.file_type().expect("inspect state test entry"); + if file_type.is_dir() { + directories.push(path); + continue; + } + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with(".tmp") + || (name.starts_with(".a3s-runtime-") && name.ends_with(".tmp")) + { + staging.push(path); + } + } + } + staging.sort(); + staging +} + +#[test] +fn subprocess_atomic_write_helper() { + let Ok(root) = std::env::var(ROOT_ENV) else { + return; + }; + let store = FileRuntimeStateStore::new(PathBuf::from(root)); + let mut record = store.load_sync(UNIT_ID).expect("load atomic-write unit"); + record.observation.observed_at_ms = NOW + 2; + record + .validate() + .expect("validate atomic-write replacement"); + let path = store.record_path(UNIT_ID, false).expect("record path"); + atomic_write(&path, &record, "state record").expect("rewrite state record"); +} + +#[test] +fn subprocess_atomic_io_fault_helper() { + let Ok(root) = std::env::var(ROOT_ENV) else { + return; + }; + let request_id = std::env::var(OPERATION_ENV).expect("fault request ID"); + let result = PathBuf::from(std::env::var(READY_ENV).expect("fault result path")); + let store = FileRuntimeStateStore::new(PathBuf::from(root)); + match store.reserve_action_sync(RuntimeActionKind::Stop, action(&request_id), NOW + 2) { + Err(RuntimeError::Transport(error)) => { + std::fs::write(result, error).expect("write injected I/O result"); + } + other => panic!("injected state I/O fault returned {other:?}"), + } +} + +#[cfg(unix)] +#[test] +fn state_crash_002_atomic_write_process_kills_preserve_a_complete_record() { + let cases = [ + ("state.atomic-write.after-create", false), + ("state.atomic-write.after-partial-write", false), + ("state.atomic-write.after-complete-write", false), + ("state.atomic-write.after-file-sync", false), + ("state.atomic-write.after-permissions", false), + ("state.atomic-write.after-publish", true), + ("state.atomic-write.after-directory-sync", true), + ]; + + for (failpoint, published) in cases { + let directory = tempfile::tempdir().expect("atomic-write state root"); + let store = FileRuntimeStateStore::new(directory.path()); + prepare_running(&store); + let ready = directory.path().join(format!("{failpoint}.ready")); + let mut child = spawn_atomic_crash_helper(directory.path(), &ready, failpoint); + kill_at_failpoint(&mut child, &ready, failpoint); + + let record = store.load_sync(UNIT_ID).expect("load post-crash record"); + assert_eq!( + record.observation.observed_at_ms, + if published { NOW + 2 } else { NOW + 1 }, + "{failpoint}: unexpected published state" + ); + let staged = staging_files(directory.path()); + if published { + assert!( + staged.is_empty(), + "{failpoint}: published state left a staging file: {staged:?}" + ); + } else { + assert!( + !staged.is_empty(), + "{failpoint}: pre-publish crash did not leave a recoverable staging file" + ); + } + + let mut recovered = record; + recovered.observation.observed_at_ms = NOW + 3; + recovered.validate().expect("validate recovered record"); + let path = store.record_path(UNIT_ID, false).expect("record path"); + atomic_write(&path, &recovered, "state record").expect("recover atomic state write"); + assert_eq!( + store + .load_sync(UNIT_ID) + .expect("load recovered state") + .observation + .observed_at_ms, + NOW + 3 + ); + assert!( + staging_files(directory.path()).is_empty(), + "{failpoint}: replay left a stale staging file" + ); + } +} + +#[cfg(unix)] +#[test] +fn state_fs_003_storage_faults_preserve_prior_state_and_reject_dispatch() { + let cases = [ + ( + "STATE-FS-003-read-only", + "state.atomic-write.before-create.read-only", + ), + ( + "STATE-FS-003-enospc", + "state.atomic-write.before-create.enospc", + ), + ( + "STATE-FS-003-inode-exhaustion", + "state.atomic-write.before-create.inode-exhaustion", + ), + ]; + + for (index, (case_id, fault)) in cases.into_iter().enumerate() { + let directory = tempfile::tempdir().expect("storage-fault state root"); + let store = FileRuntimeStateStore::new(directory.path()); + let running = prepare_running(&store); + let request_id = format!("storage-fault-{index}"); + let result = directory.path().join(format!("{request_id}.result")); + let mut child = spawn_io_fault_helper(directory.path(), &result, &request_id, fault); + wait_for_success(&mut child, case_id); + + assert!( + result.is_file(), + "{case_id}: helper did not report the fault" + ); + assert_eq!( + store + .load_sync(UNIT_ID) + .expect("load prior record") + .observation, + running, + "{case_id}: failed reservation changed the durable record" + ); + assert!(matches!( + store.load_request_sync(UNIT_ID, &request_id), + Err(RuntimeError::RequestNotFound { .. }) + )); + assert!( + staging_files(directory.path()).is_empty(), + "{case_id}: failed reservation left a staging file" + ); + } +} diff --git a/src/state/mod.rs b/src/state/mod.rs index c269adf..13071f1 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -2,7 +2,8 @@ mod file; mod record; use crate::contract::{ - RuntimeActionRequest, RuntimeApplyRequest, RuntimeObservation, RuntimeRemoval, + RuntimeActionRequest, RuntimeApplyRequest, RuntimeExecRequest, RuntimeExecResult, + RuntimeObservation, RuntimeRemoval, }; use crate::RuntimeResult; use async_trait::async_trait; @@ -13,8 +14,20 @@ pub use record::{ RuntimeStateReservation, RuntimeUnitRecord, }; +#[cfg(test)] +mod transition_tests; + +/// Owned guard for one unit's cross-process operation lease. Implementations +/// release the lease when the guard is dropped. +pub trait RuntimeOperationLease: Send {} + #[async_trait] pub trait RuntimeStateStore: Send + Sync { + async fn acquire_operation_lease( + &self, + unit_id: &str, + ) -> RuntimeResult>; + async fn reserve_apply( &self, request: &RuntimeApplyRequest, @@ -28,8 +41,20 @@ pub trait RuntimeStateStore: Send + Sync { now_ms: u64, ) -> RuntimeResult; + async fn reserve_exec( + &self, + request: &RuntimeExecRequest, + now_ms: u64, + ) -> RuntimeResult; + async fn load(&self, unit_id: &str) -> RuntimeResult; + async fn load_request( + &self, + unit_id: &str, + request_id: &str, + ) -> RuntimeResult; + async fn update_observation( &self, request_id: Option<&str>, @@ -37,4 +62,6 @@ pub trait RuntimeStateStore: Send + Sync { ) -> RuntimeResult; async fn complete_removal(&self, removal: &RuntimeRemoval) -> RuntimeResult; + + async fn complete_exec(&self, result: &RuntimeExecResult) -> RuntimeResult; } diff --git a/src/state/record.rs b/src/state/record.rs index 18adc67..3661a97 100644 --- a/src/state/record.rs +++ b/src/state/record.rs @@ -1,8 +1,8 @@ use crate::contract::{ - RuntimeActionRequest, RuntimeApplyRequest, RuntimeObservation, RuntimeRemoval, RuntimeUnitSpec, + RuntimeActionRequest, RuntimeApplyRequest, RuntimeExecRequest, RuntimeExecResult, + RuntimeObservation, RuntimeRemoval, RuntimeUnitSpec, }; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -17,6 +17,7 @@ pub enum RuntimeRequestKind { Apply, Stop, Remove, + Exec, } impl From for RuntimeRequestKind { @@ -38,75 +39,191 @@ pub enum RuntimeRequestState { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeRequestReceipt { + pub schema: String, pub request_id: String, + pub unit_id: String, + pub generation: u64, pub kind: RuntimeRequestKind, pub request_digest: String, + /// Effective absolute deadline captured when the request was first reserved. + /// + /// Exec always has a deadline because its relative timeout is converted to + /// an absolute value. Persisting it prevents a pending replay from extending + /// the original execution budget. + pub deadline_at_ms: Option, pub state: RuntimeRequestState, pub observation: Option, pub removal: Option, + pub exec_result: Option, } impl RuntimeRequestReceipt { + pub const SCHEMA: &'static str = "a3s.runtime.request-receipt.v2"; + pub(crate) fn pending_apply(request: &RuntimeApplyRequest) -> Result { - Ok(Self { - request_id: request.request_id.clone(), - kind: RuntimeRequestKind::Apply, - request_digest: request.digest()?, - state: RuntimeRequestState::Pending, - observation: None, - removal: None, - }) + Ok(Self::pending( + request.request_id.clone(), + request.spec.unit_id.clone(), + request.spec.generation, + RuntimeRequestKind::Apply, + request.digest()?, + request.deadline_at_ms, + )) } pub(crate) fn pending_action( kind: RuntimeActionKind, request: &RuntimeActionRequest, ) -> Result { - Ok(Self { - request_id: request.request_id.clone(), - kind: kind.into(), - request_digest: request.digest()?, + Ok(Self::pending( + request.request_id.clone(), + request.unit_id.clone(), + request.generation, + kind.into(), + request.digest()?, + request.deadline_at_ms, + )) + } + + pub(crate) fn pending_exec( + request: &RuntimeExecRequest, + started_at_ms: u64, + ) -> Result { + let relative_deadline = started_at_ms.saturating_add(request.timeout_ms); + let deadline_at_ms = request + .deadline_at_ms + .map_or(relative_deadline, |absolute| { + absolute.min(relative_deadline) + }); + Ok(Self::pending( + request.request_id.clone(), + request.unit_id.clone(), + request.generation, + RuntimeRequestKind::Exec, + request.digest()?, + Some(deadline_at_ms), + )) + } + + fn pending( + request_id: String, + unit_id: String, + generation: u64, + kind: RuntimeRequestKind, + request_digest: String, + deadline_at_ms: Option, + ) -> Self { + Self { + schema: Self::SCHEMA.into(), + request_id, + unit_id, + generation, + kind, + request_digest, + deadline_at_ms, state: RuntimeRequestState::Pending, observation: None, removal: None, - }) + exec_result: None, + } } pub(crate) fn complete_with_observation(&mut self, observation: RuntimeObservation) { self.state = RuntimeRequestState::Completed; self.observation = Some(observation); self.removal = None; + self.exec_result = None; } pub(crate) fn complete_with_removal(&mut self, removal: RuntimeRemoval) { self.state = RuntimeRequestState::Completed; self.observation = None; self.removal = Some(removal); + self.exec_result = None; } - fn validate(&self) -> Result<(), String> { + pub(crate) fn complete_with_exec_result(&mut self, result: RuntimeExecResult) { + self.state = RuntimeRequestState::Completed; + self.observation = None; + self.removal = None; + self.exec_result = Some(result); + } + + pub fn validate(&self) -> Result<(), String> { + if self.schema != Self::SCHEMA { + return Err(format!( + "unsupported Runtime request receipt schema {:?}", + self.schema + )); + } crate::contract::validate_id("request_id", &self.request_id, 512)?; + crate::contract::validate_id("unit_id", &self.unit_id, 512)?; + if self.generation == 0 { + return Err("Runtime request receipt generation must be positive".into()); + } + if self.deadline_at_ms == Some(0) + || (self.kind == RuntimeRequestKind::Exec && self.deadline_at_ms.is_none()) + { + return Err("Runtime request receipt deadline is invalid".into()); + } crate::contract::validate_digest(&self.request_digest)?; - match (self.kind, self.state) { - (_, RuntimeRequestState::Pending) - if self.observation.is_none() && self.removal.is_none() => - { - Ok(()) - } + match ( + self.kind, + self.state, + &self.observation, + &self.removal, + &self.exec_result, + ) { + (_, RuntimeRequestState::Pending, None, None, None) => Ok(()), ( RuntimeRequestKind::Apply | RuntimeRequestKind::Stop, RuntimeRequestState::Completed, - ) if self.observation.is_some() && self.removal.is_none() => { - self.observation.as_ref().unwrap().validate() + Some(observation), + None, + None, + ) => { + observation.validate()?; + self.validate_unit_result(&observation.unit_id, observation.generation) + } + ( + RuntimeRequestKind::Remove, + RuntimeRequestState::Completed, + None, + Some(removal), + None, + ) => { + removal.validate()?; + if removal.request_id != self.request_id { + return Err("Runtime removal receipt request identity mismatch".into()); + } + self.validate_unit_result(&removal.unit_id, removal.generation) } - (RuntimeRequestKind::Remove, RuntimeRequestState::Completed) - if self.observation.is_none() && self.removal.is_some() => - { - self.removal.as_ref().unwrap().validate() + ( + RuntimeRequestKind::Exec, + RuntimeRequestState::Completed, + None, + None, + Some(result), + ) => { + result.validate()?; + if result.request_id != self.request_id { + return Err("Runtime exec receipt request identity mismatch".into()); + } + self.validate_unit_result( + &result.observation.unit_id, + result.observation.generation, + ) } _ => Err("Runtime request receipt result does not match its kind and state".into()), } } + + fn validate_unit_result(&self, unit_id: &str, generation: u64) -> Result<(), String> { + if unit_id != self.unit_id || generation != self.generation { + return Err("Runtime request receipt result identity mismatch".into()); + } + Ok(()) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -116,22 +233,17 @@ pub struct RuntimeUnitRecord { pub spec: RuntimeUnitSpec, pub observation: RuntimeObservation, pub removed_at_ms: Option, - pub requests: BTreeMap, } impl RuntimeUnitRecord { - pub const SCHEMA: &'static str = "a3s.runtime.unit-record.v1"; + pub const SCHEMA: &'static str = "a3s.runtime.unit-record.v2"; pub(crate) fn new(request: &RuntimeApplyRequest, now_ms: u64) -> Result { - let receipt = RuntimeRequestReceipt::pending_apply(request)?; - let mut requests = BTreeMap::new(); - requests.insert(request.request_id.clone(), receipt); let record = Self { schema: Self::SCHEMA.into(), spec: request.spec.clone(), observation: RuntimeObservation::accepted(&request.spec, now_ms)?, removed_at_ms: None, - requests, }; record.validate()?; Ok(record) @@ -145,40 +257,7 @@ impl RuntimeUnitRecord { )); } self.spec.validate()?; - self.observation.validate_against(&self.spec)?; - if self.requests.len() > 10_000 { - return Err("Runtime unit record exceeds request receipt limit".into()); - } - for (request_id, receipt) in &self.requests { - receipt.validate()?; - if request_id != &receipt.request_id { - return Err("Runtime request receipt key mismatch".into()); - } - if let Some(observation) = &receipt.observation { - if observation.unit_id != self.spec.unit_id { - return Err("Runtime request receipt belongs to another unit".into()); - } - } - if let Some(removal) = &receipt.removal { - if removal.unit_id != self.spec.unit_id { - return Err("Runtime removal receipt belongs to another unit".into()); - } - } - } - Ok(()) - } - - pub fn receipt(&self, request_id: &str) -> Option<&RuntimeRequestReceipt> { - self.requests.get(request_id) - } - - pub(crate) fn receipt_mut( - &mut self, - request_id: &str, - ) -> Result<&mut RuntimeRequestReceipt, String> { - self.requests - .get_mut(request_id) - .ok_or_else(|| format!("Runtime request receipt {request_id:?} was not reserved")) + self.observation.validate_against(&self.spec) } } diff --git a/src/state/transition_tests.rs b/src/state/transition_tests.rs new file mode 100644 index 0000000..3a6f107 --- /dev/null +++ b/src/state/transition_tests.rs @@ -0,0 +1,225 @@ +use super::file::validate_transition; +use crate::contract::{ + ArtifactRef, IsolationLevel, NetworkMode, ResourceLimits, RestartPolicy, RuntimeFailure, + RuntimeHealthObservation, RuntimeHealthState, RuntimeNetworkSpec, RuntimeObservation, + RuntimeProcessSpec, RuntimeUnitClass, RuntimeUnitSpec, RuntimeUnitState, +}; +use std::collections::BTreeMap; + +const STATES: [RuntimeUnitState; 9] = [ + RuntimeUnitState::Accepted, + RuntimeUnitState::Preparing, + RuntimeUnitState::Starting, + RuntimeUnitState::Running, + RuntimeUnitState::Stopping, + RuntimeUnitState::Stopped, + RuntimeUnitState::Succeeded, + RuntimeUnitState::Failed, + RuntimeUnitState::Unknown, +]; + +fn digest(character: char) -> String { + format!("sha256:{}", character.to_string().repeat(64)) +} + +fn spec(class: RuntimeUnitClass) -> RuntimeUnitSpec { + let class_name = match class { + RuntimeUnitClass::Task => "task", + RuntimeUnitClass::Service => "service", + }; + RuntimeUnitSpec { + schema: RuntimeUnitSpec::SCHEMA.into(), + unit_id: format!("transition-{class_name}"), + generation: 1, + class, + artifact: ArtifactRef { + uri: format!( + "oci://registry.example/a3s/transition@sha256:{}", + "a".repeat(64) + ), + digest: digest('a'), + media_type: "application/vnd.oci.image.manifest.v1+json".into(), + }, + process: RuntimeProcessSpec { + command: vec!["/bin/workload".into()], + args: Vec::new(), + working_directory: None, + environment: BTreeMap::new(), + }, + mounts: Vec::new(), + secrets: Vec::new(), + network: RuntimeNetworkSpec { + mode: NetworkMode::None, + ports: Vec::new(), + }, + resources: ResourceLimits { + cpu_millis: 100, + memory_bytes: 64 * 1024 * 1024, + pids: 32, + ephemeral_storage_bytes: None, + execution_timeout_ms: (class == RuntimeUnitClass::Task).then_some(60_000), + }, + isolation: IsolationLevel::Container, + health: None, + restart: match class { + RuntimeUnitClass::Task => RestartPolicy::Never, + RuntimeUnitClass::Service => RestartPolicy::Always, + }, + outputs: Vec::new(), + semantics_profile_digest: None, + } +} + +fn observation( + spec: &RuntimeUnitSpec, + state: RuntimeUnitState, + observed_at_ms: u64, +) -> RuntimeObservation { + let provider_backed = state != RuntimeUnitState::Accepted; + RuntimeObservation { + schema: RuntimeObservation::SCHEMA.into(), + unit_id: spec.unit_id.clone(), + generation: spec.generation, + spec_digest: spec.digest().expect("valid transition specification"), + class: spec.class, + state, + provider_resource_id: provider_backed.then(|| "provider/transition/1".into()), + provider_build: provider_backed.then(|| "transition-driver/1".into()), + observed_at_ms, + started_at_ms: provider_backed.then_some(100), + finished_at_ms: state.is_terminal().then_some(observed_at_ms), + health: None, + outputs: Vec::new(), + usage: None, + evidence: None, + provider_attestation: None, + failure: (state == RuntimeUnitState::Failed).then(|| RuntimeFailure { + code: "provider_failed".into(), + message: "injected transition failure".into(), + retryable: false, + }), + } +} + +fn valid_for_class(class: RuntimeUnitClass, state: RuntimeUnitState) -> bool { + class != RuntimeUnitClass::Service || state != RuntimeUnitState::Succeeded +} + +fn expected_transition(from: RuntimeUnitState, to: RuntimeUnitState) -> bool { + if from.is_terminal() { + return false; + } + from == to + || matches!( + (from, to), + (RuntimeUnitState::Accepted, RuntimeUnitState::Preparing) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Starting) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Running) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Failed) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Accepted, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Starting) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Running) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Stopping) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Failed) + | (RuntimeUnitState::Preparing, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Starting, RuntimeUnitState::Running) + | (RuntimeUnitState::Starting, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Starting, RuntimeUnitState::Stopping) + | (RuntimeUnitState::Starting, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Starting, RuntimeUnitState::Failed) + | (RuntimeUnitState::Starting, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Running, RuntimeUnitState::Stopping) + | (RuntimeUnitState::Running, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Running, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Running, RuntimeUnitState::Failed) + | (RuntimeUnitState::Running, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Stopping, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Stopping, RuntimeUnitState::Failed) + | (RuntimeUnitState::Stopping, RuntimeUnitState::Unknown) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Preparing) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Starting) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Running) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Stopping) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Stopped) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Succeeded) + | (RuntimeUnitState::Unknown, RuntimeUnitState::Failed) + ) +} + +#[test] +fn state_transition_001_task_and_service_matrices_have_complete_oracles() { + for class in [RuntimeUnitClass::Task, RuntimeUnitClass::Service] { + let spec = spec(class); + for from in STATES + .into_iter() + .filter(|state| valid_for_class(class, *state)) + { + for to in STATES + .into_iter() + .filter(|state| valid_for_class(class, *state)) + { + let current = observation(&spec, from, 200); + let next = observation(&spec, to, 201); + let actual = validate_transition(¤t, &next, &spec).is_ok(); + assert_eq!( + actual, + expected_transition(from, to), + "STATE-TRANSITION-{class:?}-{from:?}-{to:?}" + ); + } + } + } +} + +#[test] +fn state_transition_002_terminal_identity_time_and_class_rules_fail_closed() { + for class in [RuntimeUnitClass::Task, RuntimeUnitClass::Service] { + let spec = spec(class); + for terminal in [ + RuntimeUnitState::Stopped, + RuntimeUnitState::Succeeded, + RuntimeUnitState::Failed, + ] + .into_iter() + .filter(|state| valid_for_class(class, *state)) + { + let current = observation(&spec, terminal, 200); + assert!( + validate_transition(¤t, ¤t, &spec).is_ok(), + "STATE-TERMINAL-REPLAY-{class:?}-{terminal:?}" + ); + } + } + + let task = spec(RuntimeUnitClass::Task); + let running = observation(&task, RuntimeUnitState::Running, 200); + let mut substituted = observation(&task, RuntimeUnitState::Running, 201); + substituted.provider_resource_id = Some("provider/substituted/1".into()); + assert!(validate_transition(&running, &substituted, &task).is_err()); + + let unknown = observation(&task, RuntimeUnitState::Unknown, 200); + assert!(validate_transition(&unknown, &substituted, &task).is_ok()); + let accepted = observation(&task, RuntimeUnitState::Accepted, 201); + assert!(validate_transition(&unknown, &accepted, &task).is_err()); + + let mut backwards = observation(&task, RuntimeUnitState::Running, 199); + backwards.provider_resource_id = running.provider_resource_id.clone(); + assert!(validate_transition(&running, &backwards, &task).is_err()); + + let service = spec(RuntimeUnitClass::Service); + let service_running = observation(&service, RuntimeUnitState::Running, 200); + let service_succeeded = observation(&service, RuntimeUnitState::Succeeded, 201); + assert!(validate_transition(&service_running, &service_succeeded, &service).is_err()); + + let mut task_with_health = observation(&task, RuntimeUnitState::Running, 201); + task_with_health.health = Some(RuntimeHealthObservation { + state: RuntimeHealthState::Healthy, + checked_at_ms: 201, + message: None, + }); + assert!(validate_transition(&running, &task_with_health, &task).is_err()); +} diff --git a/tests/conformance_profiles.rs b/tests/conformance_profiles.rs new file mode 100644 index 0000000..4238b3a --- /dev/null +++ b/tests/conformance_profiles.rs @@ -0,0 +1,424 @@ +#[path = "process_races/driver.rs"] +mod driver; + +use a3s_runtime::contract::{ + ArtifactRef, HealthCheckKind, IsolationLevel, MountKind, NetworkMode, ResourceControl, + ResourceLimits, RestartPolicy, RuntimeActionRequest, RuntimeApplyRequest, RuntimeFeature, + RuntimeNetworkSpec, RuntimeProcessSpec, RuntimeUnitClass, RuntimeUnitSpec, +}; +use a3s_runtime::{ + required_runtime_profiles, runtime_profile_requirements, verify_runtime_profiles, + FileRuntimeStateStore, ManagedRuntimeClient, RuntimeConformanceCase, RuntimeConformanceFixture, + RuntimeConformanceInventory, RuntimeConformanceProfile, RuntimeConformanceProfileEvidence, + RuntimeDriver, RuntimeError, RuntimeResult, +}; +use async_trait::async_trait; +use driver::{ProcessRaceDriver, IMAGE_MEDIA_TYPE}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +fn spec(unit_id: &str, class: RuntimeUnitClass, args: &[&str]) -> RuntimeUnitSpec { + RuntimeUnitSpec { + schema: RuntimeUnitSpec::SCHEMA.into(), + unit_id: unit_id.into(), + generation: 1, + class, + artifact: ArtifactRef { + uri: format!( + "oci://registry.example/a3s/conformance@sha256:{}", + "a".repeat(64) + ), + digest: format!("sha256:{}", "a".repeat(64)), + media_type: IMAGE_MEDIA_TYPE.into(), + }, + process: RuntimeProcessSpec { + command: vec!["/bin/fixture".into()], + args: args.iter().copied().map(Into::into).collect(), + working_directory: None, + environment: BTreeMap::new(), + }, + mounts: Vec::new(), + secrets: Vec::new(), + network: RuntimeNetworkSpec { + mode: NetworkMode::None, + ports: Vec::new(), + }, + resources: ResourceLimits { + cpu_millis: 100, + memory_bytes: 64 * 1024 * 1024, + pids: 32, + ephemeral_storage_bytes: None, + execution_timeout_ms: (class == RuntimeUnitClass::Task).then_some(1_000), + }, + isolation: IsolationLevel::Container, + health: None, + restart: if class == RuntimeUnitClass::Task { + RestartPolicy::Never + } else { + RestartPolicy::Always + }, + outputs: Vec::new(), + semantics_profile_digest: None, + } +} + +fn apply(request_id: &str, spec: RuntimeUnitSpec) -> RuntimeApplyRequest { + RuntimeApplyRequest { + schema: RuntimeApplyRequest::SCHEMA.into(), + request_id: request_id.into(), + deadline_at_ms: None, + spec, + } +} + +fn action(request_id: &str, apply: &RuntimeApplyRequest) -> RuntimeActionRequest { + RuntimeActionRequest { + schema: RuntimeActionRequest::SCHEMA.into(), + request_id: request_id.into(), + unit_id: apply.spec.unit_id.clone(), + generation: apply.spec.generation, + deadline_at_ms: None, + } +} + +fn base_case() -> a3s_runtime::RuntimeBaseConformanceCase { + let task_apply = apply( + "base-task-apply", + spec("base-task", RuntimeUnitClass::Task, &[]), + ); + let service_apply = apply( + "base-service-apply", + spec("base-service", RuntimeUnitClass::Service, &[]), + ); + let task_failure_apply = apply( + "base-failure-apply", + spec("base-failure", RuntimeUnitClass::Task, &["fail"]), + ); + let task_timeout_apply = apply( + "base-timeout-apply", + spec("base-timeout", RuntimeUnitClass::Task, &["timeout"]), + ); + let generation_apply = apply( + "base-generation-apply", + spec("base-generation", RuntimeUnitClass::Service, &[]), + ); + let generation_conflict_apply = apply( + "base-generation-conflict", + spec("base-generation", RuntimeUnitClass::Service, &["changed"]), + ); + a3s_runtime::RuntimeBaseConformanceCase { + lifecycle: RuntimeConformanceCase { + task_remove: action("base-task-remove", &task_apply), + service_stop: action("base-service-stop", &service_apply), + service_remove: action("base-service-remove", &service_apply), + task_apply, + service_apply, + }, + task_failure_remove: action("base-failure-remove", &task_failure_apply), + task_failure_apply, + task_timeout_remove: action("base-timeout-remove", &task_timeout_apply), + task_timeout_apply, + generation_remove: action("base-generation-remove", &generation_apply), + generation_apply, + generation_conflict_apply, + } +} + +struct ProfileFixture { + base: a3s_runtime::RuntimeBaseConformanceCase, + driver: ProcessRaceDriver, + available: BTreeSet, + incomplete: Option, + cleanup_calls: AtomicUsize, +} + +impl ProfileFixture { + fn new(provider_root: &Path) -> Self { + Self { + base: base_case(), + driver: ProcessRaceDriver::new(provider_root), + available: BTreeSet::from([ + RuntimeConformanceProfile::Recovery, + RuntimeConformanceProfile::Networking, + RuntimeConformanceProfile::Resources, + RuntimeConformanceProfile::Security, + ]), + incomplete: None, + cleanup_calls: AtomicUsize::new(0), + } + } + + fn unit_ids(&self) -> Vec { + vec![ + self.base.lifecycle.task_apply.spec.unit_id.clone(), + self.base.lifecycle.service_apply.spec.unit_id.clone(), + self.base.task_failure_apply.spec.unit_id.clone(), + self.base.task_timeout_apply.spec.unit_id.clone(), + self.base.generation_apply.spec.unit_id.clone(), + ] + } +} + +#[async_trait] +impl RuntimeConformanceFixture for ProfileFixture { + fn base_case(&self) -> &a3s_runtime::RuntimeBaseConformanceCase { + &self.base + } + + fn available_profiles(&self) -> BTreeSet { + self.available.clone() + } + + async fn inventory(&self) -> RuntimeResult { + let driver = self.driver.clone(); + let unit_ids = self.unit_ids(); + tokio::task::spawn_blocking(move || { + let mut entries = BTreeMap::new(); + for unit_id in unit_ids { + for resource in driver.inventory(&unit_id)? { + entries.insert( + resource.resource_id, + format!("{}:{:?}", resource.generation, resource.state), + ); + } + } + Ok(RuntimeConformanceInventory { entries }) + }) + .await + .map_err(|error| RuntimeError::Transport(format!("inventory task failed: {error}")))? + } + + async fn run_profile( + &self, + _client: &dyn a3s_runtime::RuntimeClient, + capabilities: &a3s_runtime::contract::RuntimeCapabilities, + profile: RuntimeConformanceProfile, + ) -> RuntimeResult { + let requirements = runtime_profile_requirements(capabilities, profile)?; + let mut evidence = RuntimeConformanceProfileEvidence { + profile, + case_ids: requirements.case_ids, + capability_claims: requirements.capability_claims, + }; + if self.incomplete == Some(profile) { + let _ = evidence.case_ids.pop_first(); + } + Ok(evidence) + } + + async fn cleanup(&self) -> RuntimeResult<()> { + self.cleanup_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +fn client(state_root: &Path, provider_root: &Path) -> ManagedRuntimeClient { + ManagedRuntimeClient::new( + Arc::new(FileRuntimeStateStore::new(state_root)), + Arc::new(ProcessRaceDriver::new(provider_root)), + ) +} + +#[tokio::test] +async fn conf_profile_001_base_and_capability_profiles_run_without_inventory_delta() { + let state = tempfile::tempdir().expect("profile state root"); + let provider = tempfile::tempdir().expect("profile provider root"); + let fixture = ProfileFixture::new(provider.path()); + let report = verify_runtime_profiles(&client(state.path(), provider.path()), &fixture) + .await + .expect("verify profile suite"); + assert_eq!(fixture.cleanup_calls.load(Ordering::SeqCst), 1); + assert_eq!(report.inventory_before, report.inventory_after); + assert_eq!( + report + .profiles + .iter() + .map(|evidence| evidence.profile) + .collect::>(), + BTreeSet::from([ + RuntimeConformanceProfile::Base, + RuntimeConformanceProfile::Recovery, + RuntimeConformanceProfile::Networking, + RuntimeConformanceProfile::Resources, + RuntimeConformanceProfile::Security, + ]) + ); +} + +#[tokio::test] +async fn conf_profile_002_missing_mandatory_profile_fails_before_provider_work() { + let state = tempfile::tempdir().expect("missing-profile state root"); + let provider = tempfile::tempdir().expect("missing-profile provider root"); + let mut fixture = ProfileFixture::new(provider.path()); + fixture + .available + .remove(&RuntimeConformanceProfile::Recovery); + assert!(matches!( + verify_runtime_profiles(&client(state.path(), provider.path()), &fixture).await, + Err(RuntimeError::Protocol(message)) if message.contains("recovery") + )); + assert_eq!(fixture.cleanup_calls.load(Ordering::SeqCst), 0); + assert!(fixture + .driver + .inventory("base-task") + .expect("missing-profile inventory") + .is_empty()); +} + +#[tokio::test] +async fn conf_profile_003_incomplete_evidence_fails_and_still_cleans_up() { + let state = tempfile::tempdir().expect("incomplete-profile state root"); + let provider = tempfile::tempdir().expect("incomplete-profile provider root"); + let mut fixture = ProfileFixture::new(provider.path()); + fixture.incomplete = Some(RuntimeConformanceProfile::Recovery); + assert!(matches!( + verify_runtime_profiles(&client(state.path(), provider.path()), &fixture).await, + Err(RuntimeError::Protocol(message)) if message.contains("evidence is incomplete") + )); + assert_eq!(fixture.cleanup_calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn conf_profile_004_all_advertised_optional_families_activate() { + let provider = tempfile::tempdir().expect("activation provider root"); + let driver = ProcessRaceDriver::new(provider.path()); + let mut capabilities = driver.capabilities().await.expect("driver capabilities"); + capabilities.mount_kinds = vec![MountKind::Volume]; + capabilities.health_check_kinds = vec![HealthCheckKind::Command]; + capabilities + .isolation_levels + .push(IsolationLevel::Confidential); + capabilities.features.extend([ + RuntimeFeature::Logs, + RuntimeFeature::Exec, + RuntimeFeature::SecretReferences, + RuntimeFeature::OutputArtifacts, + RuntimeFeature::Usage, + RuntimeFeature::Attestation, + ]); + let profiles = required_runtime_profiles(&capabilities).expect("derive profiles"); + assert_eq!( + profiles, + BTreeSet::from([ + RuntimeConformanceProfile::Base, + RuntimeConformanceProfile::Recovery, + RuntimeConformanceProfile::Networking, + RuntimeConformanceProfile::Mounts, + RuntimeConformanceProfile::Health, + RuntimeConformanceProfile::Resources, + RuntimeConformanceProfile::Logs, + RuntimeConformanceProfile::Exec, + RuntimeConformanceProfile::Security, + RuntimeConformanceProfile::Outputs, + RuntimeConformanceProfile::Evidence, + ]) + ); +} + +#[tokio::test] +async fn conf_profile_005_requirements_expand_every_advertised_behavior() { + let provider = tempfile::tempdir().expect("requirements provider root"); + let driver = ProcessRaceDriver::new(provider.path()); + let mut capabilities = driver.capabilities().await.expect("driver capabilities"); + capabilities.network_modes = vec![ + NetworkMode::None, + NetworkMode::Outbound, + NetworkMode::Service, + ]; + capabilities.mount_kinds = vec![MountKind::Volume, MountKind::Tmpfs]; + capabilities.health_check_kinds = vec![ + HealthCheckKind::Http, + HealthCheckKind::Tcp, + HealthCheckKind::Command, + ]; + capabilities.resource_controls = vec![ + ResourceControl::Cpu, + ResourceControl::Memory, + ResourceControl::Pids, + ResourceControl::ExecutionTimeout, + ]; + capabilities.features.extend([ + RuntimeFeature::Logs, + RuntimeFeature::Usage, + RuntimeFeature::Attestation, + ]); + + let networking = + runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Networking) + .expect("networking requirements"); + assert_eq!( + networking.case_ids, + BTreeSet::from([ + "NETWORK-LOOPBACK-PUBLICATION".into(), + "NETWORK-MODE-NONE".into(), + "NETWORK-MODE-OUTBOUND".into(), + "NETWORK-MODE-SERVICE".into(), + "NETWORK-OUTBOUND-ALLOWED".into(), + "NETWORK-OUTBOUND-DENIED".into(), + "NETWORK-PORT-COLLISION".into(), + "NETWORK-PROTOCOL-TCP".into(), + "NETWORK-PROTOCOL-UDP".into(), + ]) + ); + + let mounts = runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Mounts) + .expect("mount requirements"); + assert_eq!( + mounts.case_ids, + BTreeSet::from([ + "MOUNT-CLEANUP".into(), + "MOUNT-READ-ONLY".into(), + "MOUNT-TMPFS-ISOLATION".into(), + "MOUNT-VOLUME-PERSISTENCE".into(), + ]) + ); + + let health = runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Health) + .expect("health requirements"); + assert_eq!( + health.case_ids, + BTreeSet::from([ + "HEALTH-PROBE-COMMAND".into(), + "HEALTH-PROBE-HTTP".into(), + "HEALTH-PROBE-TCP".into(), + "HEALTH-PROBE-TIMEOUT".into(), + "HEALTH-START-PERIOD".into(), + "HEALTH-THRESHOLD-TRANSITION".into(), + "HEALTH-UNHEALTHY-EXIT".into(), + ]) + ); + + let resources = + runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Resources) + .expect("resource requirements"); + assert_eq!(resources.case_ids.len(), 8); + for required in [ + "RESOURCE-CPU-CONFIG", + "RESOURCE-CPU-BEHAVIOR", + "RESOURCE-MEMORY-CONFIG", + "RESOURCE-MEMORY-BEHAVIOR", + "RESOURCE-PIDS-CONFIG", + "RESOURCE-PIDS-BEHAVIOR", + "RESOURCE-EXECUTION-TIMEOUT-CONFIG", + "RESOURCE-EXECUTION-TIMEOUT-BEHAVIOR", + ] { + assert!(resources.case_ids.contains(required)); + } + + let logs = runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Logs) + .expect("log requirements"); + assert_eq!(logs.case_ids.len(), 8); + assert!(logs.case_ids.contains("LOG-SAME-TIMESTAMP")); + assert!(logs.case_ids.contains("LOG-ROTATION-GAP")); + assert!(logs.case_ids.contains("LOG-LARGE-RECORD")); + + let evidence = runtime_profile_requirements(&capabilities, RuntimeConformanceProfile::Evidence) + .expect("evidence requirements"); + assert!(evidence.case_ids.contains("EVIDENCE-USAGE-VALIDITY")); + assert!(evidence.case_ids.contains("EVIDENCE-ATTESTATION-VALIDITY")); + assert!(evidence + .case_ids + .contains("EVIDENCE-SEMANTICS-PROFILE-BINDING")); +} diff --git a/tests/deterministic_faults.rs b/tests/deterministic_faults.rs new file mode 100644 index 0000000..ec63e97 --- /dev/null +++ b/tests/deterministic_faults.rs @@ -0,0 +1,363 @@ +mod support; + +use a3s_runtime::contract::{RuntimeInspection, RuntimeLogQuery}; +use a3s_runtime::{ + FileRuntimeStateStore, ManagedRuntimeClient, RuntimeClient, RuntimeError, RuntimeRequestState, + RuntimeStateStore, +}; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use support::fault_driver::{DeterministicFaultDriver, FaultBoundary, FaultMode, FaultOperation}; +use support::fixtures::{action_request, apply_request, exec_request}; + +fn client( + directory: &tempfile::TempDir, + driver: Arc, +) -> (ManagedRuntimeClient, Arc) { + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + (ManagedRuntimeClient::new(store.clone(), driver), store) +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock after Unix epoch") + .as_millis() as u64 +} + +fn one_if(condition: bool) -> usize { + if condition { + 1 + } else { + 0 + } +} + +#[tokio::test] +async fn fault_apply_001_transport_boundaries_replay_without_duplicate_resources() { + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + let directory = tempfile::tempdir().expect("fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = format!("fault-apply-{boundary:?}").to_ascii_lowercase(); + let request = apply_request("fault-apply-request", &unit_id); + driver.arm(FaultOperation::Apply, boundary, FaultMode::Transport); + + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Apply") + )); + assert_eq!( + driver.resource_count(&unit_id), + one_if(boundary == FaultBoundary::AfterEffect) + ); + assert_eq!( + store + .load_request(&unit_id, &request.request_id) + .await + .expect("pending apply receipt") + .state, + RuntimeRequestState::Pending + ); + + let restarted = ManagedRuntimeClient::new(store.clone(), driver.clone()); + let recovered = restarted.apply(&request).await.expect("replay apply"); + let expected_resource_id = format!("provider/{unit_id}/g1"); + assert_eq!( + recovered.provider_resource_id.as_deref(), + Some(expected_resource_id.as_str()) + ); + assert_eq!(driver.resource_count(&unit_id), 1); + assert_eq!(driver.effect_count(FaultOperation::Apply), 1); + assert_eq!( + store + .load_request(&unit_id, &request.request_id) + .await + .expect("completed apply receipt") + .state, + RuntimeRequestState::Completed + ); + assert!(driver + .trace() + .iter() + .any(|event| event.operation == FaultOperation::Apply && event.boundary == boundary)); + } +} + +#[tokio::test] +async fn fault_mutation_002_stop_remove_and_exec_replay_each_provider_boundary() { + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + let directory = tempfile::tempdir().expect("stop fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = format!("fault-stop-{boundary:?}").to_ascii_lowercase(); + client + .apply(&apply_request("fault-stop-apply", &unit_id)) + .await + .expect("prepare running stop fixture"); + let request = action_request("fault-stop-request", &unit_id); + driver.arm(FaultOperation::Stop, boundary, FaultMode::Transport); + + assert!(matches!( + client.stop(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Stop") + )); + let expected_after_fault = match boundary { + FaultBoundary::BeforeEffect => a3s_runtime::contract::RuntimeUnitState::Running, + FaultBoundary::AfterEffect => a3s_runtime::contract::RuntimeUnitState::Stopped, + }; + assert_eq!( + driver.resource_state(&unit_id, 1), + Some(expected_after_fault) + ); + let restarted = ManagedRuntimeClient::new(store.clone(), driver.clone()); + let RuntimeInspection::Found { observation, .. } = + restarted.stop(&request).await.expect("replay stop") + else { + panic!("replayed stop must preserve the unit"); + }; + assert_eq!( + observation.state, + a3s_runtime::contract::RuntimeUnitState::Stopped + ); + assert_eq!(driver.effect_count(FaultOperation::Stop), 1); + assert_eq!( + store + .load_request(&unit_id, &request.request_id) + .await + .expect("completed stop receipt") + .state, + RuntimeRequestState::Completed + ); + } + + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + let directory = tempfile::tempdir().expect("remove fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = format!("fault-remove-{boundary:?}").to_ascii_lowercase(); + client + .apply(&apply_request("fault-remove-apply", &unit_id)) + .await + .expect("prepare running remove fixture"); + let request = action_request("fault-remove-request", &unit_id); + driver.arm(FaultOperation::Remove, boundary, FaultMode::Transport); + + assert!(matches!( + client.remove(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Remove") + )); + assert_eq!( + driver.resource_count(&unit_id), + one_if(boundary == FaultBoundary::BeforeEffect) + ); + let restarted = ManagedRuntimeClient::new(store.clone(), driver.clone()); + restarted.remove(&request).await.expect("replay removal"); + assert_eq!(driver.resource_count(&unit_id), 0); + assert_eq!(driver.effect_count(FaultOperation::Remove), 1); + assert_eq!( + store + .load_request(&unit_id, &request.request_id) + .await + .expect("completed remove receipt") + .state, + RuntimeRequestState::Completed + ); + } + + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + let directory = tempfile::tempdir().expect("exec fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = format!("fault-exec-{boundary:?}").to_ascii_lowercase(); + client + .apply(&apply_request("fault-exec-apply", &unit_id)) + .await + .expect("prepare running exec fixture"); + let request = exec_request("fault-exec-request", &unit_id); + driver.arm(FaultOperation::Exec, boundary, FaultMode::Transport); + + assert!(matches!( + client.exec(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Exec") + )); + assert_eq!( + driver.effect_count(FaultOperation::Exec), + one_if(boundary == FaultBoundary::AfterEffect) + ); + assert_eq!( + store + .load_request(&unit_id, &request.request_id) + .await + .expect("pending exec receipt") + .state, + RuntimeRequestState::Pending + ); + let restarted = ManagedRuntimeClient::new(store.clone(), driver.clone()); + let result = restarted.exec(&request).await.expect("replay exec"); + assert_eq!(result.stdout, "ok\n"); + assert_eq!(driver.effect_count(FaultOperation::Exec), 1); + assert_eq!( + restarted.exec(&request).await.expect("exact exec replay"), + result + ); + assert_eq!(driver.effect_count(FaultOperation::Exec), 1); + } +} + +#[tokio::test] +async fn fault_apply_003_timeout_after_effect_needs_a_new_request_to_reconcile() { + let directory = tempfile::tempdir().expect("timeout fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = "fault-apply-timeout"; + let mut request = apply_request("fault-timeout-request", unit_id); + request.deadline_at_ms = Some(now_ms() + 200); + driver.arm( + FaultOperation::Apply, + FaultBoundary::AfterEffect, + FaultMode::Hang, + ); + + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider apply") + )); + assert_eq!(driver.resource_count(unit_id), 1); + assert_eq!(driver.effect_count(FaultOperation::Apply), 1); + assert_eq!( + store + .load_request(unit_id, &request.request_id) + .await + .expect("timed-out pending receipt") + .state, + RuntimeRequestState::Pending + ); + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(_)) + )); + assert_eq!(driver.effect_count(FaultOperation::Apply), 1); + + let replacement_request = apply_request("fault-timeout-reconcile", unit_id); + let recovered = client + .apply(&replacement_request) + .await + .expect("new request reattaches timed-out provider effect"); + assert_eq!( + recovered.state, + a3s_runtime::contract::RuntimeUnitState::Running + ); + assert_eq!(driver.resource_count(unit_id), 1); + assert_eq!(driver.effect_count(FaultOperation::Apply), 1); +} + +#[tokio::test] +async fn fault_apply_004_provider_panic_releases_the_operation_lease() { + let directory = tempfile::tempdir().expect("panic fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let client = Arc::new(client); + let unit_id = "fault-apply-panic"; + let request = apply_request("fault-panic-request", unit_id); + driver.arm( + FaultOperation::Apply, + FaultBoundary::AfterEffect, + FaultMode::Panic, + ); + + let operation = { + let client = client.clone(); + let request = request.clone(); + tokio::spawn(async move { client.apply(&request).await }) + }; + assert!(operation + .await + .expect_err("provider panic must unwind") + .is_panic()); + assert_eq!(driver.resource_count(unit_id), 1); + assert_eq!( + store + .load_request(unit_id, &request.request_id) + .await + .expect("panic pending receipt") + .state, + RuntimeRequestState::Pending + ); + + let restarted = ManagedRuntimeClient::new(store, driver.clone()); + let recovered = tokio::time::timeout(Duration::from_secs(1), restarted.apply(&request)) + .await + .expect("panic retained operation lease") + .expect("replay apply after provider panic"); + assert_eq!( + recovered.state, + a3s_runtime::contract::RuntimeUnitState::Running + ); + assert_eq!(driver.effect_count(FaultOperation::Apply), 1); +} + +#[tokio::test] +async fn fault_read_005_capabilities_inspect_and_logs_fail_without_state_mutation() { + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + let directory = tempfile::tempdir().expect("capability fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = format!("fault-capability-{boundary:?}").to_ascii_lowercase(); + driver.arm(FaultOperation::Capabilities, boundary, FaultMode::Transport); + assert!(matches!( + client.apply(&apply_request("fault-capability-request", &unit_id)).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Capabilities") + )); + assert!(matches!( + store.load(&unit_id).await, + Err(RuntimeError::NotFound { .. }) + )); + assert_eq!(driver.resource_count(&unit_id), 0); + } + + let directory = tempfile::tempdir().expect("read fault state root"); + let driver = Arc::new(DeterministicFaultDriver::new()); + let (client, store) = client(&directory, driver.clone()); + let unit_id = "fault-read-unit"; + client + .apply(&apply_request("fault-read-apply", unit_id)) + .await + .expect("prepare read fault fixture"); + let expected = store.load(unit_id).await.expect("durable read fixture"); + + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + driver.arm(FaultOperation::Inspect, boundary, FaultMode::Transport); + assert!(matches!( + client.inspect(unit_id).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Inspect") + )); + assert_eq!( + store + .load(unit_id) + .await + .expect("state after inspect fault"), + expected + ); + } + + let query = RuntimeLogQuery { + schema: RuntimeLogQuery::SCHEMA.into(), + unit_id: unit_id.into(), + generation: 1, + cursor: None, + limit: 10, + stream: None, + }; + for boundary in [FaultBoundary::BeforeEffect, FaultBoundary::AfterEffect] { + driver.arm(FaultOperation::Logs, boundary, FaultMode::Transport); + assert!(matches!( + client.logs(&query).await, + Err(RuntimeError::Transport(message)) if message.contains("injected Logs") + )); + assert_eq!( + store.load(unit_id).await.expect("state after log fault"), + expected + ); + } +} diff --git a/tests/general_runtime.rs b/tests/general_runtime.rs index a1cabdd..eb01ded 100644 --- a/tests/general_runtime.rs +++ b/tests/general_runtime.rs @@ -3,19 +3,22 @@ use a3s_runtime::contract::{ ResourceControl, ResourceLimits, RestartPolicy, RuntimeActionRequest, RuntimeApplyRequest, RuntimeCapabilities, RuntimeExecRequest, RuntimeExecResult, RuntimeFeature, RuntimeHealthCheck, RuntimeHealthObservation, RuntimeHealthState, RuntimeInspection, RuntimeLogChunk, - RuntimeLogQuery, RuntimeLogStream, RuntimeNetworkSpec, RuntimeObservation, RuntimePort, - RuntimeProcessSpec, RuntimeRemoval, RuntimeUnitClass, RuntimeUnitSpec, RuntimeUnitState, - TransportProtocol, + RuntimeLogQuery, RuntimeLogStream, RuntimeNetworkSpec, RuntimeObservation, + RuntimeOutputArtifact, RuntimeOutputSpec, RuntimePort, RuntimeProcessSpec, RuntimeRemoval, + RuntimeUnitClass, RuntimeUnitSpec, RuntimeUnitState, TransportProtocol, }; use a3s_runtime::{ verify_runtime_provider, FileRuntimeStateStore, ManagedRuntimeClient, ProviderId, - RuntimeClient, RuntimeClientRegistry, RuntimeClock, RuntimeConformanceCase, RuntimeDriver, - RuntimeError, RuntimeProviderFactory, RuntimeResult, RuntimeStateStore, RuntimeUnitRecord, + RuntimeActionKind, RuntimeClient, RuntimeClientRegistry, RuntimeClock, RuntimeConformanceCase, + RuntimeDriver, RuntimeError, RuntimeOperationLease, RuntimeProviderFactory, RuntimeResult, + RuntimeStateReservation, RuntimeStateStore, RuntimeUnitRecord, }; use async_trait::async_trait; use std::collections::BTreeMap; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::sync::Notify; const NOW: u64 = 1_000; const IMAGE_MEDIA_TYPE: &str = "application/vnd.oci.image.manifest.v1+json"; @@ -29,46 +32,310 @@ impl RuntimeClock for FixedClock { } } +#[derive(Debug)] +struct ManualClock(AtomicU64); + +impl ManualClock { + fn new(now_ms: u64) -> Self { + Self(AtomicU64::new(now_ms)) + } + + fn set(&self, now_ms: u64) { + self.0.store(now_ms, Ordering::SeqCst); + } +} + +impl RuntimeClock for ManualClock { + fn now_ms(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } +} + struct TestDriver { + provider: ProviderId, capabilities: RuntimeCapabilities, + state_directories: Mutex>, apply_calls: AtomicUsize, inspect_calls: AtomicUsize, stop_calls: AtomicUsize, remove_calls: AtomicUsize, + exec_calls: AtomicUsize, + exec_effects: Mutex>, + last_exec_request: Mutex>, + exec_stdout_bytes: AtomicUsize, + exec_truncated: AtomicBool, fail_next_apply: AtomicBool, + fail_next_exec_after_effect: AtomicBool, missing_on_inspect: AtomicBool, substitute_identity: AtomicBool, unordered_logs: AtomicBool, + return_starting_on_apply: AtomicBool, + return_unknown_on_apply: AtomicBool, + return_running_on_stop: AtomicBool, + substitute_exec_spec: AtomicBool, + hang_capabilities: AtomicBool, + hang_apply: AtomicBool, + hang_exec: AtomicBool, } impl TestDriver { fn new() -> Self { + let provider = ProviderId::parse("test-runtime").unwrap(); Self { - capabilities: capabilities(), + capabilities: capabilities(provider.clone()), + provider, + state_directories: Mutex::new(Vec::new()), apply_calls: AtomicUsize::new(0), inspect_calls: AtomicUsize::new(0), stop_calls: AtomicUsize::new(0), remove_calls: AtomicUsize::new(0), + exec_calls: AtomicUsize::new(0), + exec_effects: Mutex::new(BTreeMap::new()), + last_exec_request: Mutex::new(None), + exec_stdout_bytes: AtomicUsize::new(0), + exec_truncated: AtomicBool::new(false), fail_next_apply: AtomicBool::new(false), + fail_next_exec_after_effect: AtomicBool::new(false), missing_on_inspect: AtomicBool::new(false), substitute_identity: AtomicBool::new(false), unordered_logs: AtomicBool::new(false), + return_starting_on_apply: AtomicBool::new(false), + return_unknown_on_apply: AtomicBool::new(false), + return_running_on_stop: AtomicBool::new(false), + substitute_exec_spec: AtomicBool::new(false), + hang_capabilities: AtomicBool::new(false), + hang_apply: AtomicBool::new(false), + hang_exec: AtomicBool::new(false), } } fn client(self: &Arc) -> (ManagedRuntimeClient, Arc) { let directory = tempfile::tempdir().unwrap(); - let path = directory.keep(); + let path = directory.path().to_path_buf(); + self.state_directories.lock().unwrap().push(directory); let store = Arc::new(FileRuntimeStateStore::new(path)); let client = ManagedRuntimeClient::with_clock(store.clone(), self.clone(), Arc::new(FixedClock)); (client, store) } + + fn exec_effect_count(&self) -> usize { + self.exec_effects.lock().unwrap().len() + } +} + +struct GenerationDriver { + provider: ProviderId, + resources: Mutex>>, + fail_after_create: AtomicBool, + apply_calls: AtomicUsize, +} + +impl GenerationDriver { + fn new() -> Self { + Self { + provider: ProviderId::parse("generation-runtime").unwrap(), + resources: Mutex::new(BTreeMap::new()), + fail_after_create: AtomicBool::new(false), + apply_calls: AtomicUsize::new(0), + } + } + + fn generations(&self, unit_id: &str) -> Vec { + self.resources + .lock() + .unwrap() + .get(unit_id) + .map(|resources| resources.keys().copied().collect()) + .unwrap_or_default() + } +} + +#[async_trait] +impl RuntimeDriver for GenerationDriver { + fn provider_id(&self) -> &ProviderId { + &self.provider + } + + async fn capabilities(&self) -> RuntimeResult { + Ok(capabilities(self.provider.clone())) + } + + async fn apply( + &self, + spec: &RuntimeUnitSpec, + current: &RuntimeObservation, + ) -> RuntimeResult { + self.apply_calls.fetch_add(1, Ordering::SeqCst); + let resource_id = { + let mut inventory = self.resources.lock().unwrap(); + inventory + .entry(spec.unit_id.clone()) + .or_default() + .entry(spec.generation) + .or_insert_with(|| format!("provider/{}/g{}", spec.unit_id, spec.generation)) + .clone() + }; + if self.fail_after_create.swap(false, Ordering::SeqCst) { + return Err(RuntimeError::Transport( + "ambiguous generation handoff".into(), + )); + } + self.resources + .lock() + .unwrap() + .entry(spec.unit_id.clone()) + .or_default() + .retain(|generation, _| *generation == spec.generation); + + let mut observation = current.clone(); + observation.state = match spec.class { + RuntimeUnitClass::Task => RuntimeUnitState::Succeeded, + RuntimeUnitClass::Service => RuntimeUnitState::Running, + }; + observation.provider_resource_id = Some(resource_id); + observation.provider_build = Some("generation-driver/1".into()); + observation.observed_at_ms = NOW + spec.generation; + observation.started_at_ms = Some(NOW); + observation.finished_at_ms = + (spec.class == RuntimeUnitClass::Task).then_some(observation.observed_at_ms); + observation.health = spec.health.as_ref().map(|_| RuntimeHealthObservation { + state: RuntimeHealthState::Healthy, + checked_at_ms: observation.observed_at_ms, + message: None, + }); + observation + .validate_against(spec) + .map_err(RuntimeError::Protocol)?; + Ok(observation) + } + + async fn inspect(&self, _unit: &RuntimeUnitRecord) -> RuntimeResult { + Err(RuntimeError::Protocol( + "generation test did not expect inspect".into(), + )) + } + + async fn stop( + &self, + _unit: &RuntimeUnitRecord, + _request: &RuntimeActionRequest, + ) -> RuntimeResult { + Err(RuntimeError::Protocol( + "generation test did not expect stop".into(), + )) + } + + async fn remove( + &self, + _unit: &RuntimeUnitRecord, + _request: &RuntimeActionRequest, + ) -> RuntimeResult { + Err(RuntimeError::Protocol( + "generation test did not expect remove".into(), + )) + } + + async fn logs( + &self, + _unit: &RuntimeUnitRecord, + _query: &RuntimeLogQuery, + ) -> RuntimeResult> { + Err(RuntimeError::Protocol( + "generation test did not expect logs".into(), + )) + } + + async fn exec( + &self, + _unit: &RuntimeUnitRecord, + _request: &RuntimeExecRequest, + ) -> RuntimeResult { + Err(RuntimeError::Protocol( + "generation test did not expect exec".into(), + )) + } +} + +struct SignalingStateStore { + inner: Arc, + lease_started: Arc, +} + +#[async_trait] +impl RuntimeStateStore for SignalingStateStore { + async fn acquire_operation_lease( + &self, + unit_id: &str, + ) -> RuntimeResult> { + self.lease_started.notify_one(); + self.inner.acquire_operation_lease(unit_id).await + } + + async fn reserve_apply( + &self, + request: &RuntimeApplyRequest, + now_ms: u64, + ) -> RuntimeResult { + self.inner.reserve_apply(request, now_ms).await + } + + async fn reserve_action( + &self, + kind: RuntimeActionKind, + request: &RuntimeActionRequest, + now_ms: u64, + ) -> RuntimeResult { + self.inner.reserve_action(kind, request, now_ms).await + } + + async fn reserve_exec( + &self, + request: &RuntimeExecRequest, + now_ms: u64, + ) -> RuntimeResult { + self.inner.reserve_exec(request, now_ms).await + } + + async fn load(&self, unit_id: &str) -> RuntimeResult { + self.inner.load(unit_id).await + } + + async fn load_request( + &self, + unit_id: &str, + request_id: &str, + ) -> RuntimeResult { + self.inner.load_request(unit_id, request_id).await + } + + async fn update_observation( + &self, + request_id: Option<&str>, + observation: &RuntimeObservation, + ) -> RuntimeResult { + self.inner.update_observation(request_id, observation).await + } + + async fn complete_removal(&self, removal: &RuntimeRemoval) -> RuntimeResult { + self.inner.complete_removal(removal).await + } + + async fn complete_exec(&self, result: &RuntimeExecResult) -> RuntimeResult { + self.inner.complete_exec(result).await + } } #[async_trait] impl RuntimeDriver for TestDriver { + fn provider_id(&self) -> &ProviderId { + &self.provider + } + async fn capabilities(&self) -> RuntimeResult { + if self.hang_capabilities.load(Ordering::SeqCst) { + return std::future::pending::>().await; + } Ok(self.capabilities.clone()) } @@ -78,27 +345,62 @@ impl RuntimeDriver for TestDriver { current: &RuntimeObservation, ) -> RuntimeResult { self.apply_calls.fetch_add(1, Ordering::SeqCst); + if self.hang_apply.load(Ordering::SeqCst) { + return std::future::pending::>().await; + } if self.fail_next_apply.swap(false, Ordering::SeqCst) { return Err(RuntimeError::Transport("ambiguous apply".into())); } let mut observation = current.clone(); - observation.state = match spec.class { - RuntimeUnitClass::Task => RuntimeUnitState::Succeeded, - RuntimeUnitClass::Service => RuntimeUnitState::Running, + observation.state = if self.return_unknown_on_apply.load(Ordering::SeqCst) { + RuntimeUnitState::Unknown + } else if self.return_starting_on_apply.load(Ordering::SeqCst) { + RuntimeUnitState::Starting + } else { + match spec.class { + RuntimeUnitClass::Task => RuntimeUnitState::Succeeded, + RuntimeUnitClass::Service => RuntimeUnitState::Running, + } }; if self.substitute_identity.load(Ordering::SeqCst) { observation.unit_id = "substituted".into(); } - observation.provider_resource_id = Some(format!("provider/{}", spec.unit_id)); - observation.provider_build = Some("test-driver/1".into()); + if observation.state != RuntimeUnitState::Unknown + || !self.return_unknown_on_apply.load(Ordering::SeqCst) + { + let provider_identity = if current.state == RuntimeUnitState::Unknown { + format!("provider/recovered/{}", spec.unit_id) + } else { + format!("provider/{}", spec.unit_id) + }; + observation.provider_resource_id = Some(provider_identity); + observation.provider_build = Some("test-driver/1".into()); + } observation.observed_at_ms = NOW + 1; observation.started_at_ms = Some(NOW); - observation.finished_at_ms = (spec.class == RuntimeUnitClass::Task).then_some(NOW + 1); + observation.finished_at_ms = observation.state.is_terminal().then_some(NOW + 1); observation.health = spec.health.as_ref().map(|_| RuntimeHealthObservation { state: RuntimeHealthState::Healthy, checked_at_ms: NOW + 1, message: None, }); + observation.outputs = if observation.state == RuntimeUnitState::Succeeded { + spec.outputs + .iter() + .map(|expected| RuntimeOutputArtifact { + name: expected.name.clone(), + artifact: ArtifactRef { + media_type: expected.media_type.clone(), + ..artifact('b') + }, + size_bytes: expected.max_bytes.min(4), + }) + .collect() + } else { + Vec::new() + }; + observation.provider_attestation = + (spec.isolation == IsolationLevel::Confidential).then(|| artifact('c')); Ok(observation) } @@ -106,6 +408,7 @@ impl RuntimeDriver for TestDriver { self.inspect_calls.fetch_add(1, Ordering::SeqCst); if self.missing_on_inspect.load(Ordering::SeqCst) { return Ok(RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), unit_id: unit.spec.unit_id.clone(), last_generation: Some(unit.spec.generation), }); @@ -113,6 +416,7 @@ impl RuntimeDriver for TestDriver { let mut observation = unit.observation.clone(); observation.observed_at_ms += 1; Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), observation: Box::new(observation), }) } @@ -124,10 +428,19 @@ impl RuntimeDriver for TestDriver { ) -> RuntimeResult { self.stop_calls.fetch_add(1, Ordering::SeqCst); let mut observation = unit.observation.clone(); - observation.state = RuntimeUnitState::Stopped; + observation.state = if self.return_running_on_stop.load(Ordering::SeqCst) { + RuntimeUnitState::Running + } else { + RuntimeUnitState::Stopped + }; observation.observed_at_ms += 1; - observation.finished_at_ms = Some(observation.observed_at_ms); - observation.health = None; + observation.finished_at_ms = observation + .state + .is_terminal() + .then_some(observation.observed_at_ms); + if observation.state.is_terminal() { + observation.health = None; + } observation.outputs.clear(); Ok(observation) } @@ -160,6 +473,7 @@ impl RuntimeDriver for TestDriver { }; Ok(vec![ RuntimeLogChunk { + schema: RuntimeLogChunk::SCHEMA.into(), cursor: "cursor-1".into(), sequence: 1, observed_at_ms: NOW, @@ -167,6 +481,7 @@ impl RuntimeDriver for TestDriver { data: "started\n".into(), }, RuntimeLogChunk { + schema: RuntimeLogChunk::SCHEMA.into(), cursor: "cursor-2".into(), sequence: second_sequence, observed_at_ms: NOW + 1, @@ -181,14 +496,49 @@ impl RuntimeDriver for TestDriver { unit: &RuntimeUnitRecord, request: &RuntimeExecRequest, ) -> RuntimeResult { - Ok(RuntimeExecResult { + self.exec_calls.fetch_add(1, Ordering::SeqCst); + *self.last_exec_request.lock().unwrap() = Some(request.clone()); + if self.hang_exec.load(Ordering::SeqCst) { + return std::future::pending::>().await; + } + let effect_key = format!( + "{}/{}/{}", + request.unit_id, request.generation, request.request_id + ); + if let Some(result) = self.exec_effects.lock().unwrap().get(&effect_key).cloned() { + return Ok(result); + } + let mut observation = unit.observation.clone(); + if self.substitute_exec_spec.load(Ordering::SeqCst) { + observation.spec_digest = digest('f'); + } + let stdout_bytes = self.exec_stdout_bytes.load(Ordering::SeqCst); + let result = RuntimeExecResult { + schema: RuntimeExecResult::SCHEMA.into(), request_id: request.request_id.clone(), - observation: unit.observation.clone(), + observation, exit_code: 0, - stdout: "ok\n".into(), + stdout: if stdout_bytes == 0 { + "ok\n".into() + } else { + "x".repeat(stdout_bytes) + }, stderr: String::new(), - truncated: false, - }) + truncated: self.exec_truncated.load(Ordering::SeqCst), + }; + self.exec_effects + .lock() + .unwrap() + .insert(effect_key, result.clone()); + if self + .fail_next_exec_after_effect + .swap(false, Ordering::SeqCst) + { + return Err(RuntimeError::Transport( + "ambiguous exec acknowledgement".into(), + )); + } + Ok(result) } } @@ -212,7 +562,7 @@ fn resources(timeout: Option) -> ResourceLimits { cpu_millis: 500, memory_bytes: 128 * 1024 * 1024, pids: 128, - ephemeral_storage_bytes: 1024 * 1024 * 1024, + ephemeral_storage_bytes: Some(1024 * 1024 * 1024), execution_timeout_ms: timeout, } } @@ -292,10 +642,10 @@ fn action(request_id: &str, unit_id: &str, generation: u64) -> RuntimeActionRequ } } -fn capabilities() -> RuntimeCapabilities { +fn capabilities(provider_id: ProviderId) -> RuntimeCapabilities { RuntimeCapabilities { schema: RuntimeCapabilities::SCHEMA.into(), - provider_id: "test-runtime".into(), + provider_id, provider_build: "test-driver/1".into(), unit_classes: vec![RuntimeUnitClass::Task, RuntimeUnitClass::Service], artifact_media_types: vec![IMAGE_MEDIA_TYPE.into()], @@ -324,6 +674,7 @@ fn capabilities() -> RuntimeCapabilities { RuntimeFeature::Remove, RuntimeFeature::Logs, RuntimeFeature::Exec, + RuntimeFeature::OutputArtifacts, ], } } @@ -364,7 +715,11 @@ async fn ambiguous_apply_is_reentered_with_the_same_identity() { let pending = store.load("service-ambiguous").await.unwrap(); assert_eq!(pending.observation.state, RuntimeUnitState::Accepted); assert_eq!( - pending.receipt("apply-ambiguous").unwrap().state, + store + .load_request("service-ambiguous", "apply-ambiguous") + .await + .unwrap() + .state, a3s_runtime::RuntimeRequestState::Pending ); @@ -410,6 +765,76 @@ async fn request_and_generation_conflicts_fail_closed() { )); } +#[tokio::test] +async fn generation_reconciliation_retires_the_previous_resource_on_success() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(GenerationDriver::new()); + let client = ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + + client + .apply(&apply( + "generation-success-one", + service("generation-success", 1), + )) + .await + .unwrap(); + assert_eq!(driver.generations("generation-success"), vec![1]); + + let observation = client + .apply(&apply( + "generation-success-two", + service("generation-success", 2), + )) + .await + .unwrap(); + assert_eq!(observation.generation, 2); + assert_eq!(driver.generations("generation-success"), vec![2]); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn generation_reconciliation_recovers_create_before_error_after_restart() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(GenerationDriver::new()); + let client = + ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), Arc::new(FixedClock)); + client + .apply(&apply( + "generation-recovery-one", + service("generation-recovery", 1), + )) + .await + .unwrap(); + + let second = apply("generation-recovery-two", service("generation-recovery", 2)); + driver.fail_after_create.store(true, Ordering::SeqCst); + assert!(matches!( + client.apply(&second).await, + Err(RuntimeError::Transport(message)) if message.contains("ambiguous generation handoff") + )); + assert_eq!(driver.generations("generation-recovery"), vec![1, 2]); + assert_eq!( + store + .load_request("generation-recovery", "generation-recovery-two") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); + + drop(client); + let restarted = ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + let recovered = restarted.apply(&second).await.unwrap(); + assert_eq!(recovered.generation, 2); + assert_eq!(driver.generations("generation-recovery"), vec![2]); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 3); + + assert_eq!(restarted.apply(&second).await.unwrap(), recovered); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 3); +} + #[tokio::test] async fn stop_remove_and_absence_are_durable_and_idempotent() { let driver = Arc::new(TestDriver::new()); @@ -423,7 +848,7 @@ async fn stop_remove_and_absence_are_durable_and_idempotent() { let first_stop = client.stop(&stop).await.unwrap(); assert!(matches!( &first_stop, - RuntimeInspection::Found { observation } + RuntimeInspection::Found { observation, .. } if observation.state == RuntimeUnitState::Stopped )); assert_eq!(client.stop(&stop).await.unwrap(), first_stop); @@ -444,6 +869,7 @@ async fn stop_remove_and_absence_are_durable_and_idempotent() { assert_eq!( client.inspect("service-lifecycle").await.unwrap(), RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), unit_id: "service-lifecycle".into(), last_generation: Some(1), } @@ -470,98 +896,1063 @@ async fn unsupported_capabilities_fail_before_state_or_provider_work() { } #[tokio::test] -async fn provider_identity_substitution_and_unordered_logs_are_rejected() { - let driver = Arc::new(TestDriver::new()); - driver.substitute_identity.store(true, Ordering::SeqCst); - let (client, _) = driver.client(); - assert!(matches!( - client - .apply(&apply("substitute", service("identity-service", 1))) - .await, - Err(RuntimeError::Protocol(_)) - )); +async fn optional_resource_controls_gate_only_workloads_that_request_them() { + let mut driver = TestDriver::new(); + driver + .capabilities + .resource_controls + .retain(|control| *control != ResourceControl::EphemeralStorage); + let driver = Arc::new(driver); + let (client, store) = driver.client(); - driver.substitute_identity.store(false, Ordering::SeqCst); + let mut without_quota = service("without-ephemeral-quota", 1); + without_quota.resources.ephemeral_storage_bytes = None; client - .apply(&apply("identity-retry", service("identity-service", 1))) + .apply(&apply("without-ephemeral-quota", without_quota)) .await - .unwrap(); - driver.unordered_logs.store(true, Ordering::SeqCst); + .expect("unrelated workload must remain supported"); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); + assert!(matches!( client - .logs(&RuntimeLogQuery { - unit_id: "identity-service".into(), - generation: 1, - cursor: None, - limit: 100, - stream: None, - }) + .apply(&apply( + "with-ephemeral-quota", + service("with-ephemeral-quota", 1), + )) .await, - Err(RuntimeError::Protocol(_)) + Err(RuntimeError::UnsupportedCapabilities(missing)) + if missing == vec!["resource_control:EphemeralStorage"] )); + assert!(matches!( + store.load("with-ephemeral-quota").await, + Err(RuntimeError::NotFound { .. }) + )); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); } #[tokio::test] -async fn log_and_exec_capabilities_use_the_active_generation() { - let driver = Arc::new(TestDriver::new()); - let (client, _) = driver.client(); - client - .apply(&apply("apply-tools", service("service-tools", 1))) - .await - .unwrap(); - let logs = client - .logs(&RuntimeLogQuery { - unit_id: "service-tools".into(), - generation: 1, - cursor: None, - limit: 100, - stream: None, - }) - .await - .unwrap(); - assert_eq!(logs.len(), 2); - - let result = client - .exec(&RuntimeExecRequest { - request_id: "exec-tools".into(), - unit_id: "service-tools".into(), - generation: 1, - command: vec!["/bin/true".into()], - timeout_ms: 1_000, - }) - .await - .unwrap(); - assert_eq!(result.exit_code, 0); +async fn provider_identity_mismatch_fails_before_state_or_provider_work() { + let mut driver = TestDriver::new(); + driver.capabilities.provider_id = ProviderId::parse("substituted-provider").unwrap(); + let driver = Arc::new(driver); + let (client, store) = driver.client(); assert!(matches!( client - .logs(&RuntimeLogQuery { - unit_id: "service-tools".into(), - generation: 2, - cursor: None, - limit: 100, - stream: None, - }) + .apply(&apply("provider-mismatch", service("provider-mismatch", 1))) .await, - Err(RuntimeError::GenerationConflict { .. }) + Err(RuntimeError::Protocol(message)) if message.contains("reported capabilities") )); + assert!(matches!( + store.load("provider-mismatch").await, + Err(RuntimeError::NotFound { .. }) + )); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 0); } #[tokio::test] -async fn expired_requests_never_reserve_or_dispatch() { - let driver = Arc::new(TestDriver::new()); - let (client, store) = driver.client(); - let mut request = apply("expired", service("expired-service", 1)); - request.deadline_at_ms = Some(NOW); +async fn output_and_confidential_capabilities_fail_closed_and_outputs_are_exact() { + let mut output_task = task("output-task", 1); + output_task.outputs = vec![RuntimeOutputSpec { + name: "result".into(), + path: "/outputs/result.json".into(), + media_type: "application/json".into(), + max_bytes: 16, + }]; + + let mut unsupported = TestDriver::new(); + unsupported + .capabilities + .features + .retain(|feature| *feature != RuntimeFeature::OutputArtifacts); + let unsupported = Arc::new(unsupported); + let (client, store) = unsupported.client(); assert!(matches!( - client.apply(&request).await, - Err(RuntimeError::DeadlineExceeded(_)) + client + .apply(&apply("output-unsupported", output_task.clone())) + .await, + Err(RuntimeError::UnsupportedCapabilities(missing)) + if missing == vec!["feature:OutputArtifacts"] )); assert!(matches!( - store.load("expired-service").await, + store.load("output-task").await, Err(RuntimeError::NotFound { .. }) )); -} + + output_task.unit_id = "output-task-supported".into(); + let supported = Arc::new(TestDriver::new()); + let (client, _) = supported.client(); + let observation = client + .apply(&apply("output-supported", output_task.clone())) + .await + .unwrap(); + assert_eq!(observation.outputs.len(), 1); + assert_eq!(observation.outputs[0].name, "result"); + assert!(observation.outputs[0].size_bytes <= 16); + let mut oversized = observation; + oversized.outputs[0].size_bytes = 17; + assert!(oversized.validate_against(&output_task).is_err()); + + let mut confidential_driver = TestDriver::new(); + confidential_driver + .capabilities + .isolation_levels + .push(IsolationLevel::Confidential); + let confidential_driver = Arc::new(confidential_driver); + let (client, store) = confidential_driver.client(); + let mut confidential = service("confidential-service", 1); + confidential.isolation = IsolationLevel::Confidential; + assert!(matches!( + client + .apply(&apply("confidential-unsupported", confidential)) + .await, + Err(RuntimeError::UnsupportedCapabilities(missing)) + if missing == vec!["feature:Attestation"] + )); + assert!(matches!( + store.load("confidential-service").await, + Err(RuntimeError::NotFound { .. }) + )); + + let mut supported_confidential = TestDriver::new(); + supported_confidential + .capabilities + .isolation_levels + .push(IsolationLevel::Confidential); + supported_confidential + .capabilities + .features + .push(RuntimeFeature::Attestation); + let supported_confidential = Arc::new(supported_confidential); + let (client, _) = supported_confidential.client(); + let mut confidential = service("confidential-supported", 1); + confidential.isolation = IsolationLevel::Confidential; + let observation = client + .apply(&apply("confidential-supported", confidential)) + .await + .unwrap(); + assert!(observation.provider_attestation.is_some()); +} + +#[tokio::test] +async fn apply_stop_and_exec_postconditions_fail_closed() { + let driver = Arc::new(TestDriver::new()); + driver + .return_starting_on_apply + .store(true, Ordering::SeqCst); + let (client, _) = driver.client(); + let spec = service("postconditions", 1); + assert!(matches!( + client + .apply(&apply("postconditions-starting", spec.clone())) + .await, + Err(RuntimeError::Protocol(message)) if message.contains("invalid Service result") + )); + + driver + .return_starting_on_apply + .store(false, Ordering::SeqCst); + client + .apply(&apply("postconditions-running", spec)) + .await + .unwrap(); + driver.return_running_on_stop.store(true, Ordering::SeqCst); + let stop = action("postconditions-stop", "postconditions", 1); + assert!(matches!( + client.stop(&stop).await, + Err(RuntimeError::Protocol(message)) if message.contains("nonterminal state") + )); + + driver.return_running_on_stop.store(false, Ordering::SeqCst); + client.stop(&stop).await.unwrap(); + assert!(matches!( + client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-stopped".into(), + unit_id: "postconditions".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }) + .await, + Err(RuntimeError::InvalidRequest(message)) if message.contains("running unit") + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 0); + + let unknown_driver = Arc::new(TestDriver::new()); + unknown_driver + .return_unknown_on_apply + .store(true, Ordering::SeqCst); + let (unknown_client, _) = unknown_driver.client(); + assert!(matches!( + unknown_client + .apply(&apply( + "postconditions-unknown", + service("postconditions-unknown", 1), + )) + .await, + Err(RuntimeError::Protocol(message)) + if message.contains("without provider identity") + )); +} + +#[tokio::test] +async fn exec_result_must_bind_the_complete_unit_spec() { + let driver = Arc::new(TestDriver::new()); + let (client, _) = driver.client(); + client + .apply(&apply("exec-binding-apply", service("exec-binding", 1))) + .await + .unwrap(); + driver.substitute_exec_spec.store(true, Ordering::SeqCst); + + assert!(matches!( + client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-binding-request".into(), + unit_id: "exec-binding".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }) + .await, + Err(RuntimeError::Protocol(message)) + if message.contains("does not match the unit specification") + )); +} + +#[tokio::test] +async fn completed_exec_replays_durably_after_client_restart_and_rejects_conflicts() { + let driver = Arc::new(TestDriver::new()); + let (client, store) = driver.client(); + client + .apply(&apply("exec-restart-apply", service("exec-restart", 1))) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-restart-request".into(), + unit_id: "exec-restart".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }; + let expected = client.exec(&request).await.unwrap(); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + assert_eq!( + store + .load_request("exec-restart", "exec-restart-request") + .await + .unwrap() + .exec_result, + Some(expected.clone()) + ); + + let restarted = ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + assert_eq!(restarted.exec(&request).await.unwrap(), expected); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + + let mut conflicting = request; + conflicting.command = vec!["/bin/false".into()]; + assert!(matches!( + restarted.exec(&conflicting).await, + Err(RuntimeError::RequestConflict { .. }) + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn pending_exec_reattaches_after_an_ambiguous_after_effect_error() { + let driver = Arc::new(TestDriver::new()); + let (client, store) = driver.client(); + client + .apply(&apply("exec-ambiguous-apply", service("exec-ambiguous", 1))) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-ambiguous-request".into(), + unit_id: "exec-ambiguous".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }; + driver + .fail_next_exec_after_effect + .store(true, Ordering::SeqCst); + + assert!(matches!( + client.exec(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("ambiguous exec") + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + assert_eq!(driver.exec_effect_count(), 1); + assert_eq!( + store + .load_request("exec-ambiguous", "exec-ambiguous-request") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); + + let restarted = + ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), Arc::new(FixedClock)); + let recovered = restarted.exec(&request).await.unwrap(); + assert_eq!(recovered.stdout, "ok\n"); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 2); + assert_eq!(driver.exec_effect_count(), 1); + assert_eq!( + store + .load_request("exec-ambiguous", "exec-ambiguous-request") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Completed + ); + assert_eq!(restarted.exec(&request).await.unwrap(), recovered); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 2); + assert_eq!(driver.exec_effect_count(), 1); +} + +#[tokio::test] +async fn cancelled_exec_releases_the_lease_and_remains_replayable() { + let driver = Arc::new(TestDriver::new()); + let (client, store) = driver.client(); + let client = Arc::new(client); + client + .apply(&apply("exec-cancel-apply", service("exec-cancel", 1))) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-cancel-request".into(), + unit_id: "exec-cancel".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 10_000, + deadline_at_ms: None, + }; + driver.hang_exec.store(true, Ordering::SeqCst); + let operation = { + let client = client.clone(); + let request = request.clone(); + tokio::spawn(async move { client.exec(&request).await }) + }; + tokio::time::timeout(Duration::from_secs(1), async { + while driver.exec_calls.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("exec did not reach the provider before cancellation"); + operation.abort(); + assert!(operation.await.unwrap_err().is_cancelled()); + assert_eq!(driver.exec_effect_count(), 0); + assert_eq!( + store + .load_request("exec-cancel", "exec-cancel-request") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); + + driver.hang_exec.store(false, Ordering::SeqCst); + let recovered = tokio::time::timeout(Duration::from_secs(1), client.exec(&request)) + .await + .expect("cancelled exec retained its operation lease") + .unwrap(); + assert_eq!(recovered.stdout, "ok\n"); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 2); + assert_eq!(driver.exec_effect_count(), 1); +} + +#[tokio::test] +async fn maximum_exec_output_and_truncation_are_durable() { + let driver = Arc::new(TestDriver::new()); + driver + .exec_stdout_bytes + .store(16 * 1024 * 1024, Ordering::SeqCst); + driver.exec_truncated.store(true, Ordering::SeqCst); + let (client, store) = driver.client(); + client + .apply(&apply("exec-large-apply", service("exec-large", 1))) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-large-request".into(), + unit_id: "exec-large".into(), + generation: 1, + command: vec!["/bin/large-output".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }; + let result = client.exec(&request).await.unwrap(); + assert_eq!(result.stdout.len(), 16 * 1024 * 1024); + assert!(result.truncated); + result.validate().unwrap(); + assert_eq!(driver.exec_effect_count(), 1); + + let restarted = ManagedRuntimeClient::with_clock(store, driver.clone(), Arc::new(FixedClock)); + assert_eq!(restarted.exec(&request).await.unwrap(), result); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + assert_eq!(driver.exec_effect_count(), 1); + + let mut oversized = result; + oversized.stdout.push('x'); + assert!(oversized.validate().is_err()); +} + +#[tokio::test] +async fn lc_exec_002_oversized_provider_result_stays_pending() { + let driver = Arc::new(TestDriver::new()); + driver + .exec_stdout_bytes + .store(16 * 1024 * 1024 + 1, Ordering::SeqCst); + let (client, store) = driver.client(); + client + .apply(&apply("exec-oversized-apply", service("exec-oversized", 1))) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-oversized-request".into(), + unit_id: "exec-oversized".into(), + generation: 1, + command: vec!["/bin/oversized-output".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }; + + for expected_calls in [1, 2] { + assert!(matches!( + client.exec(&request).await, + Err(RuntimeError::Protocol(message)) + if message.contains("exec output exceeds protocol limits") + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), expected_calls); + assert_eq!(driver.exec_effect_count(), 1); + let receipt = store + .load_request("exec-oversized", "exec-oversized-request") + .await + .unwrap(); + assert_eq!(receipt.state, a3s_runtime::RuntimeRequestState::Pending); + assert!(receipt.exec_result.is_none()); + } +} + +#[tokio::test] +async fn completed_mutations_replay_after_deadline_provider_loss_and_removal() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let clock = Arc::new(ManualClock::new(NOW)); + let client = ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); + let deadline = NOW + 1_000; + + let mut apply_request = apply("durable-replay-apply", service("durable-replay", 1)); + apply_request.deadline_at_ms = Some(deadline); + let applied = client.apply(&apply_request).await.unwrap(); + + let exec_request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "durable-replay-exec".into(), + unit_id: "durable-replay".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 500, + deadline_at_ms: Some(deadline), + }; + let executed = client.exec(&exec_request).await.unwrap(); + + let mut stop_request = action("durable-replay-stop", "durable-replay", 1); + stop_request.deadline_at_ms = Some(deadline); + let stopped = client.stop(&stop_request).await.unwrap(); + + let mut remove_request = action("durable-replay-remove", "durable-replay", 1); + remove_request.deadline_at_ms = Some(deadline); + let removed = client.remove(&remove_request).await.unwrap(); + + clock.set(deadline); + driver.hang_capabilities.store(true, Ordering::SeqCst); + let restarted = ManagedRuntimeClient::with_clock(store, driver.clone(), clock); + + assert_eq!(restarted.apply(&apply_request).await.unwrap(), applied); + assert_eq!(restarted.exec(&exec_request).await.unwrap(), executed); + assert_eq!(restarted.stop(&stop_request).await.unwrap(), stopped); + assert_eq!(restarted.remove(&remove_request).await.unwrap(), removed); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + assert_eq!(driver.stop_calls.load(Ordering::SeqCst), 1); + assert_eq!(driver.remove_calls.load(Ordering::SeqCst), 1); + + let mut conflicting = apply_request; + conflicting.spec.process.command = vec!["/bin/false".into()]; + assert!(matches!( + restarted.apply(&conflicting).await, + Err(RuntimeError::RequestConflict { .. }) + )); +} + +#[tokio::test] +async fn expired_pending_request_remains_pending_and_is_not_redispatched() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let clock = Arc::new(ManualClock::new(NOW)); + let client = ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); + let mut request = apply("pending-expiry", service("pending-expiry", 1)); + let deadline = NOW + 1_000; + request.deadline_at_ms = Some(deadline); + driver.hang_apply.store(true, Ordering::SeqCst); + + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider apply") + )); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); + assert_eq!( + store + .load_request("pending-expiry", "pending-expiry") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); + + clock.set(deadline); + driver.hang_apply.store(false, Ordering::SeqCst); + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(_)) + )); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); + assert_eq!( + store + .load_request("pending-expiry", "pending-expiry") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); +} + +#[test] +fn top_level_log_exec_and_inspection_records_require_current_schemas() { + let mut query = RuntimeLogQuery { + schema: RuntimeLogQuery::SCHEMA.into(), + unit_id: "schema-unit".into(), + generation: 1, + cursor: None, + limit: 1, + stream: None, + }; + query.validate().unwrap(); + query.schema = "a3s.runtime.log-query.v0".into(); + assert!(query.validate().is_err()); + + let mut chunk = RuntimeLogChunk { + schema: RuntimeLogChunk::SCHEMA.into(), + cursor: "cursor".into(), + sequence: 1, + observed_at_ms: NOW, + stream: RuntimeLogStream::Stdout, + data: String::new(), + }; + chunk.validate().unwrap(); + chunk.schema.clear(); + assert!(chunk.validate().is_err()); + + let mut exec = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "schema-exec".into(), + unit_id: "schema-unit".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1, + deadline_at_ms: None, + }; + exec.validate().unwrap(); + exec.schema = "future".into(); + assert!(exec.validate().is_err()); + + let inspection = RuntimeInspection::NotFound { + schema: "future".into(), + unit_id: "schema-unit".into(), + last_generation: None, + }; + assert!(inspection.validate().is_err()); + + assert!( + serde_json::from_value::(serde_json::json!({ + "unit_id": "schema-unit", + "generation": 1, + "cursor": null, + "limit": 1, + "stream": null + })) + .is_err() + ); +} + +#[tokio::test] +async fn provider_identity_substitution_and_unordered_logs_are_rejected() { + let driver = Arc::new(TestDriver::new()); + driver.substitute_identity.store(true, Ordering::SeqCst); + let (client, _) = driver.client(); + assert!(matches!( + client + .apply(&apply("substitute", service("identity-service", 1))) + .await, + Err(RuntimeError::Protocol(_)) + )); + + driver.substitute_identity.store(false, Ordering::SeqCst); + client + .apply(&apply("identity-retry", service("identity-service", 1))) + .await + .unwrap(); + driver.unordered_logs.store(true, Ordering::SeqCst); + assert!(matches!( + client + .logs(&RuntimeLogQuery { + schema: RuntimeLogQuery::SCHEMA.into(), + unit_id: "identity-service".into(), + generation: 1, + cursor: None, + limit: 100, + stream: None, + }) + .await, + Err(RuntimeError::Protocol(_)) + )); +} + +#[tokio::test] +async fn log_and_exec_capabilities_use_the_active_generation() { + let driver = Arc::new(TestDriver::new()); + let (client, _) = driver.client(); + client + .apply(&apply("apply-tools", service("service-tools", 1))) + .await + .unwrap(); + let logs = client + .logs(&RuntimeLogQuery { + schema: RuntimeLogQuery::SCHEMA.into(), + unit_id: "service-tools".into(), + generation: 1, + cursor: None, + limit: 100, + stream: None, + }) + .await + .unwrap(); + assert_eq!(logs.len(), 2); + + let result = client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-tools".into(), + unit_id: "service-tools".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }) + .await + .unwrap(); + assert_eq!(result.exit_code, 0); + assert_eq!( + client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-tools".into(), + unit_id: "service-tools".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }) + .await + .unwrap(), + result + ); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + + assert!(matches!( + client + .logs(&RuntimeLogQuery { + schema: RuntimeLogQuery::SCHEMA.into(), + unit_id: "service-tools".into(), + generation: 2, + cursor: None, + limit: 100, + stream: None, + }) + .await, + Err(RuntimeError::GenerationConflict { .. }) + )); +} + +#[tokio::test] +async fn expired_requests_never_reserve_or_dispatch() { + let driver = Arc::new(TestDriver::new()); + let (client, store) = driver.client(); + let mut request = apply("expired", service("expired-service", 1)); + request.deadline_at_ms = Some(NOW); + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(_)) + )); + assert!(matches!( + store.load("expired-service").await, + Err(RuntimeError::NotFound { .. }) + )); +} + +#[tokio::test] +async fn file_operation_leases_serialize_one_unit_but_not_different_units() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let first = store.acquire_operation_lease("lease-unit").await.unwrap(); + + assert!(tokio::time::timeout( + Duration::from_millis(20), + store.acquire_operation_lease("lease-unit"), + ) + .await + .is_err()); + let different = tokio::time::timeout( + Duration::from_millis(100), + store.acquire_operation_lease("different-unit"), + ) + .await + .expect("a different unit must not wait behind the held lease") + .unwrap(); + + drop(different); + drop(first); +} + +#[tokio::test] +async fn subprocess_operation_lease_helper() { + let Ok(root) = std::env::var("A3S_RUNTIME_LEASE_HELPER_ROOT") else { + return; + }; + let unit_id = std::env::var("A3S_RUNTIME_LEASE_HELPER_UNIT").unwrap(); + let ready = std::env::var("A3S_RUNTIME_LEASE_HELPER_READY").unwrap(); + let release = std::env::var("A3S_RUNTIME_LEASE_HELPER_RELEASE").unwrap(); + let store = FileRuntimeStateStore::new(root); + let _lease = store.acquire_operation_lease(&unit_id).await.unwrap(); + std::fs::write(&ready, b"ready").unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + while !std::path::Path::new(&release).is_file() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("parent did not release the subprocess lease"); +} + +#[tokio::test] +async fn file_operation_lease_serializes_independent_processes() { + use std::process::Command; + + let directory = tempfile::tempdir().unwrap(); + let ready = directory.path().join("helper.ready"); + let release = directory.path().join("helper.release"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("subprocess_operation_lease_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env("A3S_RUNTIME_LEASE_HELPER_ROOT", directory.path()) + .env("A3S_RUNTIME_LEASE_HELPER_UNIT", "cross-process-unit") + .env("A3S_RUNTIME_LEASE_HELPER_READY", &ready) + .env("A3S_RUNTIME_LEASE_HELPER_RELEASE", &release) + .spawn() + .unwrap(); + + tokio::time::timeout(Duration::from_secs(10), async { + while !ready.is_file() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("subprocess did not acquire the operation lease"); + + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let waiting_store = store.clone(); + let waiter = tokio::spawn(async move { + waiting_store + .acquire_operation_lease("cross-process-unit") + .await + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!waiter.is_finished()); + let different = tokio::time::timeout( + Duration::from_millis(100), + store.acquire_operation_lease("cross-process-other"), + ) + .await + .expect("different-unit cross-process lease was blocked") + .unwrap(); + drop(different); + + std::fs::write(&release, b"release").unwrap(); + let acquired = tokio::time::timeout(Duration::from_secs(10), waiter) + .await + .expect("same-unit waiter did not acquire after subprocess release") + .unwrap() + .unwrap(); + drop(acquired); + assert!(child.wait().unwrap().success()); +} + +#[tokio::test] +async fn deadline_is_rechecked_after_the_operation_lease_wait() { + let directory = tempfile::tempdir().unwrap(); + let file_store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let held = file_store + .acquire_operation_lease("post-lease-deadline") + .await + .unwrap(); + let lease_started = Arc::new(Notify::new()); + let state: Arc = Arc::new(SignalingStateStore { + inner: file_store.clone(), + lease_started: lease_started.clone(), + }); + let driver = Arc::new(TestDriver::new()); + let clock = Arc::new(ManualClock::new(NOW)); + let client = ManagedRuntimeClient::with_clock(state, driver.clone(), clock.clone()); + let mut request = apply("post-lease-deadline", service("post-lease-deadline", 1)); + request.deadline_at_ms = Some(NOW + 1_000); + + let operation = tokio::spawn(async move { client.apply(&request).await }); + lease_started.notified().await; + clock.set(NOW + 1_000); + drop(held); + + assert!(matches!( + operation.await.unwrap(), + Err(RuntimeError::DeadlineExceeded(message)) + if message.contains("before provider dispatch") + )); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 0); + assert!(matches!( + file_store.load("post-lease-deadline").await, + Err(RuntimeError::NotFound { .. }) + )); +} + +#[tokio::test] +async fn deadlines_bound_capability_queries_and_provider_dispatch() { + let capability_driver = Arc::new(TestDriver::new()); + capability_driver + .hang_capabilities + .store(true, Ordering::SeqCst); + let (client, store) = capability_driver.client(); + let mut request = apply("capability-timeout", service("capability-timeout", 1)); + request.deadline_at_ms = Some(NOW + 10); + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(message)) + if message.contains("capability query") + )); + assert!(matches!( + store.load("capability-timeout").await, + Err(RuntimeError::NotFound { .. }) + )); + assert_eq!(capability_driver.apply_calls.load(Ordering::SeqCst), 0); + + let provider_driver = Arc::new(TestDriver::new()); + provider_driver.hang_apply.store(true, Ordering::SeqCst); + let (client, store) = provider_driver.client(); + let mut request = apply("provider-timeout", service("provider-timeout", 1)); + request.deadline_at_ms = Some(NOW + 10); + assert!(matches!( + client.apply(&request).await, + Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider apply") + )); + let pending = store + .load_request("provider-timeout", "provider-timeout") + .await + .unwrap(); + assert_eq!(pending.state, a3s_runtime::RuntimeRequestState::Pending); + assert_eq!(provider_driver.apply_calls.load(Ordering::SeqCst), 1); + + provider_driver.hang_apply.store(false, Ordering::SeqCst); + assert_eq!( + client.apply(&request).await.unwrap().state, + RuntimeUnitState::Running + ); + assert_eq!(provider_driver.apply_calls.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn exec_uses_the_smaller_relative_or_absolute_deadline() { + let driver = Arc::new(TestDriver::new()); + let (client, _) = driver.client(); + client + .apply(&apply("exec-deadline-apply", service("exec-deadline", 1))) + .await + .unwrap(); + driver.hang_exec.store(true, Ordering::SeqCst); + + assert!(matches!( + client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-relative-deadline".into(), + unit_id: "exec-deadline".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 10, + deadline_at_ms: Some(NOW + 1_000), + }) + .await, + Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider exec") + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + + assert!(matches!( + client + .exec(&RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-absolute-deadline".into(), + unit_id: "exec-deadline".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: Some(NOW), + }) + .await, + Err(RuntimeError::DeadlineExceeded(_)) + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn lc_exec_003_provider_receives_the_durable_effective_deadline() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let clock = Arc::new(ManualClock::new(NOW)); + let client = ManagedRuntimeClient::with_clock(store, driver.clone(), clock.clone()); + client + .apply(&apply( + "exec-provider-deadline-apply", + service("exec-provider-deadline", 1), + )) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-provider-deadline-request".into(), + unit_id: "exec-provider-deadline".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + }; + driver + .fail_next_exec_after_effect + .store(true, Ordering::SeqCst); + + assert!(matches!( + client.exec(&request).await, + Err(RuntimeError::Transport(message)) if message.contains("ambiguous") + )); + assert_eq!( + driver + .last_exec_request + .lock() + .unwrap() + .as_ref() + .unwrap() + .deadline_at_ms, + Some(NOW + 1_000) + ); + + clock.set(NOW + 200); + let replayed = client.exec(&request).await.unwrap(); + assert_eq!(replayed.request_id, request.request_id); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 2); + assert_eq!(driver.exec_effect_count(), 1); + assert_eq!( + driver + .last_exec_request + .lock() + .unwrap() + .as_ref() + .unwrap() + .deadline_at_ms, + Some(NOW + 1_000) + ); +} + +#[tokio::test] +async fn lc_exec_001_pending_replay_cannot_extend_original_relative_deadline() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let clock = Arc::new(ManualClock::new(NOW)); + let client = ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock.clone()); + client + .apply(&apply( + "exec-relative-replay-apply", + service("exec-relative-replay", 1), + )) + .await + .unwrap(); + let request = RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: "exec-relative-replay-request".into(), + unit_id: "exec-relative-replay".into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 10, + deadline_at_ms: None, + }; + driver.hang_exec.store(true, Ordering::SeqCst); + + assert!(matches!( + client.exec(&request).await, + Err(RuntimeError::DeadlineExceeded(message)) if message.contains("provider exec") + )); + let pending = store + .load_request("exec-relative-replay", "exec-relative-replay-request") + .await + .unwrap(); + assert_eq!(pending.state, a3s_runtime::RuntimeRequestState::Pending); + assert_eq!(pending.deadline_at_ms, Some(NOW + 10)); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + + clock.set(NOW + 10); + driver.hang_exec.store(false, Ordering::SeqCst); + let restarted = ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), clock); + assert!(matches!( + restarted.exec(&request).await, + Err(RuntimeError::DeadlineExceeded(_)) + )); + assert_eq!(driver.exec_calls.load(Ordering::SeqCst), 1); + assert_eq!( + store + .load_request("exec-relative-replay", "exec-relative-replay-request") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); +} #[tokio::test] async fn provider_loss_becomes_a_durable_unknown_observation() { @@ -574,7 +1965,7 @@ async fn provider_loss_becomes_a_durable_unknown_observation() { driver.missing_on_inspect.store(true, Ordering::SeqCst); let inspection = client.inspect("service-loss").await.unwrap(); - let RuntimeInspection::Found { observation } = inspection else { + let RuntimeInspection::Found { observation, .. } = inspection else { panic!("a previously observed provider unit must not become definitively absent"); }; assert_eq!(observation.state, RuntimeUnitState::Unknown); @@ -589,6 +1980,39 @@ async fn provider_loss_becomes_a_durable_unknown_observation() { ); } +#[tokio::test] +async fn same_generation_apply_recovers_a_lost_provider_unit_once() { + let driver = Arc::new(TestDriver::new()); + let (client, store) = driver.client(); + let spec = service("service-recovery", 1); + let running = client + .apply(&apply("apply-recovery", spec.clone())) + .await + .unwrap(); + + driver.missing_on_inspect.store(true, Ordering::SeqCst); + let RuntimeInspection::Found { observation, .. } = + client.inspect("service-recovery").await.unwrap() + else { + panic!("provider loss must first become a durable unknown observation"); + }; + assert_eq!(observation.state, RuntimeUnitState::Unknown); + + driver.missing_on_inspect.store(false, Ordering::SeqCst); + let recovery = apply("reapply-recovery", spec); + let recovered = client.apply(&recovery).await.unwrap(); + assert_eq!(recovered.state, RuntimeUnitState::Running); + assert_ne!(recovered.provider_resource_id, running.provider_resource_id); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 2); + + assert_eq!(client.apply(&recovery).await.unwrap(), recovered); + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 2); + assert_eq!( + store.load("service-recovery").await.unwrap().observation, + recovered + ); +} + #[tokio::test] async fn terminal_observations_are_immutable() { let driver = Arc::new(TestDriver::new()); @@ -630,11 +2054,58 @@ async fn concurrent_file_reservations_preserve_every_request() { let record = store.load("concurrent-service").await.unwrap(); assert_eq!(record.spec, expected); - assert_eq!(record.requests.len(), 32); - assert!(record - .requests - .values() - .all(|receipt| receipt.state == a3s_runtime::RuntimeRequestState::Pending)); + for index in 0..32 { + assert_eq!( + store + .load_request("concurrent-service", &format!("apply-{index}")) + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Pending + ); + } +} + +#[tokio::test] +async fn request_journal_preserves_more_than_ten_thousand_exact_replays() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let client = + ManagedRuntimeClient::with_clock(store.clone(), driver.clone(), Arc::new(FixedClock)); + let spec = service("journal-capacity", 1); + client + .apply(&apply("journal-request-0", spec.clone())) + .await + .unwrap(); + for index in 1..=10_000 { + client + .apply(&apply(&format!("journal-request-{index}"), spec.clone())) + .await + .unwrap(); + } + + assert_eq!(driver.apply_calls.load(Ordering::SeqCst), 1); + assert_eq!( + store + .load_request("journal-capacity", "journal-request-10000") + .await + .unwrap() + .state, + a3s_runtime::RuntimeRequestState::Completed + ); + let unit_directory = std::fs::read_dir(directory.path().join("units")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + assert_eq!( + std::fs::read_dir(unit_directory.join("requests")) + .unwrap() + .count(), + 10_001 + ); } struct CountingFactory { @@ -643,20 +2114,24 @@ struct CountingFactory { client: Arc, } +#[async_trait] impl RuntimeProviderFactory for CountingFactory { fn provider_id(&self) -> &ProviderId { &self.provider } - fn create(&self) -> RuntimeResult> { + async fn create(&self) -> RuntimeResult> { self.creates.fetch_add(1, Ordering::SeqCst); Ok(self.client.clone()) } } -#[test] -fn provider_registry_never_falls_back_or_replaces_a_factory() { - let driver = Arc::new(TestDriver::new()); +#[tokio::test] +async fn provider_registry_never_falls_back_or_replaces_a_factory() { + let mut driver = TestDriver::new(); + driver.provider = ProviderId::parse("explicit-provider").unwrap(); + driver.capabilities.provider_id = driver.provider.clone(); + let driver = Arc::new(driver); let (client, _) = driver.client(); let factory = Arc::new(CountingFactory { provider: ProviderId::parse("explicit-provider").unwrap(), @@ -668,20 +2143,40 @@ fn provider_registry_never_falls_back_or_replaces_a_factory() { let missing = ProviderId::parse("missing-provider").unwrap(); assert!(matches!( - registry.connect(&missing), + registry.connect(&missing).await, Err(RuntimeError::ProviderUnavailable(_)) )); assert_eq!(factory.creates.load(Ordering::SeqCst), 0); assert!(registry.register(factory.clone()).is_err()); assert_eq!(factory.creates.load(Ordering::SeqCst), 0); - registry.connect(factory.provider_id()).unwrap(); + registry.connect(factory.provider_id()).await.unwrap(); + assert_eq!(factory.creates.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn provider_registry_rejects_a_factory_client_identity_mismatch() { + let driver = Arc::new(TestDriver::new()); + let (client, _) = driver.client(); + let factory = Arc::new(CountingFactory { + provider: ProviderId::parse("registered-provider").unwrap(), + creates: AtomicUsize::new(0), + client: Arc::new(client), + }); + let mut registry = RuntimeClientRegistry::new(); + registry.register(factory.clone()).unwrap(); + + assert!(matches!( + registry.connect(factory.provider_id()).await, + Err(RuntimeError::Protocol(message)) if message.contains("created client reporting") + )); assert_eq!(factory.creates.load(Ordering::SeqCst), 1); } #[cfg(unix)] #[tokio::test] async fn file_state_store_rejects_symbolic_link_boundaries() { + use sha2::{Digest, Sha256}; use std::os::unix::fs::symlink; let directory = tempfile::tempdir().unwrap(); @@ -705,12 +2200,9 @@ async fn file_state_store_rejects_symbolic_link_boundaries() { .reserve_apply(&apply("secure", task("secure-unit", 1)), NOW) .await .unwrap(); - let lock_path = std::fs::read_dir(root.join("locks")) - .unwrap() - .next() - .unwrap() - .unwrap() - .path(); + let lock_path = root + .join("locks") + .join(format!("{:x}.lock", Sha256::digest(b"secure-unit"))); std::fs::remove_file(&lock_path).unwrap(); let lock_target = root.join("lock-target"); std::fs::write(&lock_target, b"do not follow").unwrap(); @@ -721,6 +2213,91 @@ async fn file_state_store_rejects_symbolic_link_boundaries() { )); } +#[cfg(unix)] +#[tokio::test] +async fn request_journal_layout_is_versioned_owner_only_and_fail_closed() { + use sha2::{Digest, Sha256}; + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileRuntimeStateStore::new(directory.path())); + let driver = Arc::new(TestDriver::new()); + let client = ManagedRuntimeClient::with_clock(store.clone(), driver, Arc::new(FixedClock)); + client + .apply(&apply("layout-request", service("layout-unit", 1))) + .await + .unwrap(); + + let unit_directory = std::fs::read_dir(directory.path().join("units")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let record_path = unit_directory.join("record.json"); + let receipt_path = std::fs::read_dir(unit_directory.join("requests")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let record: serde_json::Value = + serde_json::from_slice(&std::fs::read(&record_path).unwrap()).unwrap(); + let receipt: serde_json::Value = + serde_json::from_slice(&std::fs::read(&receipt_path).unwrap()).unwrap(); + assert_eq!(record["schema"], a3s_runtime::RuntimeUnitRecord::SCHEMA); + assert!(record.get("requests").is_none()); + assert_eq!( + receipt["schema"], + a3s_runtime::RuntimeRequestReceipt::SCHEMA + ); + assert_eq!( + std::fs::metadata(&unit_directory) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); + assert_eq!( + std::fs::metadata(&record_path) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + assert_eq!( + std::fs::metadata(&receipt_path) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + + let mut unknown_field = receipt; + unknown_field["unexpected"] = serde_json::json!(true); + std::fs::write(&receipt_path, serde_json::to_vec(&unknown_field).unwrap()).unwrap(); + assert!(matches!( + store.load_request("layout-unit", "layout-request").await, + Err(RuntimeError::Protocol(_)) + )); + + let legacy = tempfile::tempdir().unwrap(); + let units = legacy.path().join("units"); + std::fs::create_dir(&units).unwrap(); + let key = format!("{:x}", Sha256::digest(b"legacy-unit")); + let legacy_record = units.join(format!("{key}.json")); + std::fs::write(&legacy_record, b"{}").unwrap(); + std::fs::set_permissions(&legacy_record, std::fs::Permissions::from_mode(0o600)).unwrap(); + let legacy_store = FileRuntimeStateStore::new(legacy.path()); + assert!(matches!( + legacy_store.load("legacy-unit").await, + Err(RuntimeError::Protocol(message)) if message.contains("explicit migration") + )); +} + #[tokio::test] async fn exported_conformance_suite_exercises_task_and_service_lifecycles() { let driver = Arc::new(TestDriver::new()); diff --git a/tests/golden/action-request-v1.json b/tests/golden/action-request-v1.json new file mode 100644 index 0000000..a12568d --- /dev/null +++ b/tests/golden/action-request-v1.json @@ -0,0 +1,7 @@ +{ + "schema": "a3s.runtime.action-request.v1", + "request_id": "golden-stop", + "unit_id": "golden-service", + "generation": 3, + "deadline_at_ms": 200000 +} diff --git a/tests/golden/apply-request-v1.json b/tests/golden/apply-request-v1.json new file mode 100644 index 0000000..8b2fc78 --- /dev/null +++ b/tests/golden/apply-request-v1.json @@ -0,0 +1,42 @@ +{ + "schema": "a3s.runtime.apply-request.v1", + "request_id": "golden-apply", + "deadline_at_ms": 200000, + "spec": { + "schema": "a3s.runtime.unit-spec.v2", + "unit_id": "golden-task", + "generation": 7, + "class": "task", + "artifact": { + "uri": "oci://registry.example/a3s/golden@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "media_type": "application/vnd.oci.image.manifest.v1+json" + }, + "process": { + "command": [], + "args": [], + "working_directory": null, + "environment": {} + }, + "mounts": [], + "secrets": [], + "network": { + "mode": "outbound", + "ports": [] + }, + "resources": { + "cpu_millis": 500, + "memory_bytes": 134217728, + "pids": 128, + "ephemeral_storage_bytes": null, + "execution_timeout_ms": 60000 + }, + "isolation": "container", + "health": null, + "restart": { + "kind": "never" + }, + "outputs": [], + "semantics_profile_digest": null + } +} diff --git a/tests/golden/capabilities-v3.json b/tests/golden/capabilities-v3.json new file mode 100644 index 0000000..c6cb8ae --- /dev/null +++ b/tests/golden/capabilities-v3.json @@ -0,0 +1,13 @@ +{ + "schema": "a3s.runtime.capabilities.v3", + "provider_id": "golden-runtime", + "provider_build": "golden-runtime/1.2.3", + "unit_classes": ["task", "service"], + "artifact_media_types": ["application/vnd.oci.image.manifest.v1+json"], + "isolation_levels": ["process", "container", "sandbox", "confidential"], + "network_modes": ["none", "outbound", "service"], + "mount_kinds": ["artifact", "volume", "tmpfs"], + "health_check_kinds": ["http", "tcp", "command"], + "resource_controls": ["cpu", "memory", "pids", "ephemeral_storage", "execution_timeout"], + "features": ["durable_identity", "stop", "remove", "logs", "exec", "usage", "attestation", "secret_references", "output_artifacts"] +} diff --git a/tests/golden/exec-request-v1.json b/tests/golden/exec-request-v1.json new file mode 100644 index 0000000..dab9aef --- /dev/null +++ b/tests/golden/exec-request-v1.json @@ -0,0 +1,9 @@ +{ + "schema": "a3s.runtime.exec-request.v1", + "request_id": "golden-exec", + "unit_id": "golden-service", + "generation": 3, + "command": ["/bin/sh", "-c", "printf ready"], + "timeout_ms": 5000, + "deadline_at_ms": 200000 +} diff --git a/tests/golden/exec-result-v1.json b/tests/golden/exec-result-v1.json new file mode 100644 index 0000000..f5e4002 --- /dev/null +++ b/tests/golden/exec-result-v1.json @@ -0,0 +1,31 @@ +{ + "schema": "a3s.runtime.exec-result.v1", + "request_id": "golden-exec", + "observation": { + "schema": "a3s.runtime.observation.v2", + "unit_id": "golden-service", + "generation": 3, + "spec_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "class": "service", + "state": "running", + "provider_resource_id": "golden/service/3", + "provider_build": "golden-runtime/1.2.3", + "observed_at_ms": 120000, + "started_at_ms": 100000, + "finished_at_ms": null, + "health": { + "state": "healthy", + "checked_at_ms": 119000, + "message": "ready" + }, + "outputs": [], + "usage": null, + "evidence": null, + "provider_attestation": null, + "failure": null + }, + "exit_code": 0, + "stdout": "ready", + "stderr": "", + "truncated": false +} diff --git a/tests/golden/inspection-found-v1.json b/tests/golden/inspection-found-v1.json new file mode 100644 index 0000000..c63bba2 --- /dev/null +++ b/tests/golden/inspection-found-v1.json @@ -0,0 +1,27 @@ +{ + "status": "found", + "schema": "a3s.runtime.inspection.v1", + "observation": { + "schema": "a3s.runtime.observation.v2", + "unit_id": "golden-service", + "generation": 3, + "spec_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "class": "service", + "state": "running", + "provider_resource_id": "golden/service/3", + "provider_build": "golden-runtime/1.2.3", + "observed_at_ms": 120000, + "started_at_ms": 100000, + "finished_at_ms": null, + "health": { + "state": "healthy", + "checked_at_ms": 119000, + "message": "ready" + }, + "outputs": [], + "usage": null, + "evidence": null, + "provider_attestation": null, + "failure": null + } +} diff --git a/tests/golden/inspection-not-found-v1.json b/tests/golden/inspection-not-found-v1.json new file mode 100644 index 0000000..b4f12b5 --- /dev/null +++ b/tests/golden/inspection-not-found-v1.json @@ -0,0 +1,6 @@ +{ + "status": "not_found", + "schema": "a3s.runtime.inspection.v1", + "unit_id": "golden-service", + "last_generation": 3 +} diff --git a/tests/golden/log-chunk-v1.json b/tests/golden/log-chunk-v1.json new file mode 100644 index 0000000..2a97847 --- /dev/null +++ b/tests/golden/log-chunk-v1.json @@ -0,0 +1,8 @@ +{ + "schema": "a3s.runtime.log-chunk.v1", + "cursor": "cursor-42", + "sequence": 42, + "observed_at_ms": 120000, + "stream": "stderr", + "data": "warning\n" +} diff --git a/tests/golden/log-query-v1.json b/tests/golden/log-query-v1.json new file mode 100644 index 0000000..5b5ad95 --- /dev/null +++ b/tests/golden/log-query-v1.json @@ -0,0 +1,8 @@ +{ + "schema": "a3s.runtime.log-query.v1", + "unit_id": "golden-service", + "generation": 3, + "cursor": "cursor-41", + "limit": 100, + "stream": "stderr" +} diff --git a/tests/golden/observation-v2.json b/tests/golden/observation-v2.json new file mode 100644 index 0000000..fadf034 --- /dev/null +++ b/tests/golden/observation-v2.json @@ -0,0 +1,49 @@ +{ + "schema": "a3s.runtime.observation.v2", + "unit_id": "golden-task", + "generation": 7, + "spec_digest": "sha256:3309c80ee072c47311be878a7fcde9818e484e9d186fa1c6eb78033fe5ef4d11", + "class": "task", + "state": "succeeded", + "provider_resource_id": "golden/task/7", + "provider_build": "golden-runtime/1.2.3", + "observed_at_ms": 120000, + "started_at_ms": 100000, + "finished_at_ms": 120000, + "health": null, + "outputs": [ + { + "name": "result", + "artifact": { + "uri": "artifact://outputs/result@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "media_type": "application/json" + }, + "size_bytes": 1024 + } + ], + "usage": { + "wall_time_ms": 20000, + "cpu_time_ms": 10000, + "peak_memory_bytes": 67108864, + "network_rx_bytes": 1024, + "network_tx_bytes": 2048, + "storage_read_bytes": 4096, + "storage_write_bytes": 8192 + }, + "evidence": { + "provider_build": "golden-runtime/1.2.3", + "spec_digest": "sha256:3309c80ee072c47311be878a7fcde9818e484e9d186fa1c6eb78033fe5ef4d11", + "semantics_profile_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "claims": { + "isolation": "sandbox", + "runtime": "golden-runtime" + } + }, + "provider_attestation": { + "uri": "attestation://golden/task/7@sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "media_type": "application/vnd.a3s.attestation.v1+json" + }, + "failure": null +} diff --git a/tests/golden/removal-v1.json b/tests/golden/removal-v1.json new file mode 100644 index 0000000..029ee77 --- /dev/null +++ b/tests/golden/removal-v1.json @@ -0,0 +1,8 @@ +{ + "schema": "a3s.runtime.removal.v1", + "request_id": "golden-remove", + "unit_id": "golden-service", + "generation": 3, + "removed_at_ms": 150000, + "already_absent": false +} diff --git a/tests/golden/request-receipt-v2.json b/tests/golden/request-receipt-v2.json new file mode 100644 index 0000000..a210564 --- /dev/null +++ b/tests/golden/request-receipt-v2.json @@ -0,0 +1,43 @@ +{ + "schema": "a3s.runtime.request-receipt.v2", + "request_id": "golden-exec", + "unit_id": "golden-service", + "generation": 3, + "kind": "exec", + "request_digest": "sha256:861f6cad44261c49a89b5cd12920ca3977d833b5c9143d748db945a08939bad0", + "deadline_at_ms": 200000, + "state": "completed", + "observation": null, + "removal": null, + "exec_result": { + "schema": "a3s.runtime.exec-result.v1", + "request_id": "golden-exec", + "observation": { + "schema": "a3s.runtime.observation.v2", + "unit_id": "golden-service", + "generation": 3, + "spec_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "class": "service", + "state": "running", + "provider_resource_id": "golden/service/3", + "provider_build": "golden-runtime/1.2.3", + "observed_at_ms": 120000, + "started_at_ms": 100000, + "finished_at_ms": null, + "health": { + "state": "healthy", + "checked_at_ms": 119000, + "message": "ready" + }, + "outputs": [], + "usage": null, + "evidence": null, + "provider_attestation": null, + "failure": null + }, + "exit_code": 0, + "stdout": "ready", + "stderr": "", + "truncated": false + } +} diff --git a/tests/golden/unit-record-v2.json b/tests/golden/unit-record-v2.json new file mode 100644 index 0000000..c6fbb02 --- /dev/null +++ b/tests/golden/unit-record-v2.json @@ -0,0 +1,60 @@ +{ + "schema": "a3s.runtime.unit-record.v2", + "spec": { + "schema": "a3s.runtime.unit-spec.v2", + "unit_id": "golden-task", + "generation": 7, + "class": "task", + "artifact": { + "uri": "oci://registry.example/a3s/golden@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "media_type": "application/vnd.oci.image.manifest.v1+json" + }, + "process": { + "command": [], + "args": [], + "working_directory": null, + "environment": {} + }, + "mounts": [], + "secrets": [], + "network": { + "mode": "outbound", + "ports": [] + }, + "resources": { + "cpu_millis": 500, + "memory_bytes": 134217728, + "pids": 128, + "ephemeral_storage_bytes": null, + "execution_timeout_ms": 60000 + }, + "isolation": "container", + "health": null, + "restart": { + "kind": "never" + }, + "outputs": [], + "semantics_profile_digest": null + }, + "observation": { + "schema": "a3s.runtime.observation.v2", + "unit_id": "golden-task", + "generation": 7, + "spec_digest": "sha256:3309c80ee072c47311be878a7fcde9818e484e9d186fa1c6eb78033fe5ef4d11", + "class": "task", + "state": "accepted", + "provider_resource_id": null, + "provider_build": null, + "observed_at_ms": 100000, + "started_at_ms": null, + "finished_at_ms": null, + "health": null, + "outputs": [], + "usage": null, + "evidence": null, + "provider_attestation": null, + "failure": null + }, + "removed_at_ms": null +} diff --git a/tests/golden/unit-spec-v2.json b/tests/golden/unit-spec-v2.json new file mode 100644 index 0000000..eeb1513 --- /dev/null +++ b/tests/golden/unit-spec-v2.json @@ -0,0 +1,97 @@ +{ + "schema": "a3s.runtime.unit-spec.v2", + "unit_id": "golden-task", + "generation": 7, + "class": "task", + "artifact": { + "uri": "oci://registry.example/a3s/golden@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "media_type": "application/vnd.oci.image.manifest.v1+json" + }, + "process": { + "command": ["/bin/sh"], + "args": ["-c", "echo ready"], + "working_directory": "/workspace", + "environment": { + "LANG": "C.UTF-8" + } + }, + "mounts": [ + { + "name": "source", + "source": { + "kind": "artifact", + "artifact": { + "uri": "artifact://source@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "media_type": "application/vnd.a3s.source.v1+tar" + } + }, + "target": "/inputs/source", + "read_only": true + }, + { + "name": "cache", + "source": { + "kind": "volume", + "volume_id": "cache-volume" + }, + "target": "/cache", + "read_only": false + }, + { + "name": "scratch", + "source": { + "kind": "tmpfs", + "size_bytes": 1048576 + }, + "target": "/tmp", + "read_only": false + } + ], + "secrets": [ + { + "name": "api token", + "reference": "secret://runtime/api-token", + "target": { + "kind": "environment", + "variable": "API_TOKEN" + } + }, + { + "name": "certificate", + "reference": "secret://runtime/certificate", + "target": { + "kind": "file", + "path": "/run/secrets/cert.pem", + "mode": 384 + } + } + ], + "network": { + "mode": "outbound", + "ports": [] + }, + "resources": { + "cpu_millis": 500, + "memory_bytes": 134217728, + "pids": 128, + "ephemeral_storage_bytes": null, + "execution_timeout_ms": 60000 + }, + "isolation": "sandbox", + "health": null, + "restart": { + "kind": "on_failure", + "max_retries": 2 + }, + "outputs": [ + { + "name": "result", + "path": "/outputs/result.json", + "media_type": "application/json", + "max_bytes": 4096 + } + ], + "semantics_profile_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" +} diff --git a/tests/process_races.rs b/tests/process_races.rs new file mode 100644 index 0000000..70a33f0 --- /dev/null +++ b/tests/process_races.rs @@ -0,0 +1,531 @@ +#[path = "process_races/driver.rs"] +mod driver; + +use a3s_runtime::contract::{ + ArtifactRef, IsolationLevel, NetworkMode, ResourceLimits, RestartPolicy, RuntimeActionRequest, + RuntimeApplyRequest, RuntimeNetworkSpec, RuntimeProcessSpec, RuntimeUnitClass, RuntimeUnitSpec, + RuntimeUnitState, +}; +use a3s_runtime::{ + FileRuntimeStateStore, ManagedRuntimeClient, RuntimeClient, RuntimeError, RuntimeRequestState, + RuntimeStateStore, +}; +use driver::{ProcessRaceDriver, ProviderResource, IMAGE_MEDIA_TYPE}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant}; + +const STATE_ROOT_ENV: &str = "A3S_RUNTIME_PROCESS_STATE_ROOT"; +const PROVIDER_ROOT_ENV: &str = "A3S_RUNTIME_PROCESS_PROVIDER_ROOT"; +const OPERATION_ENV: &str = "A3S_RUNTIME_PROCESS_OPERATION"; +const REQUEST_ID_ENV: &str = "A3S_RUNTIME_PROCESS_REQUEST_ID"; +const UNIT_ID_ENV: &str = "A3S_RUNTIME_PROCESS_UNIT_ID"; +const GENERATION_ENV: &str = "A3S_RUNTIME_PROCESS_GENERATION"; +const START_GATE_ENV: &str = "A3S_RUNTIME_PROCESS_START_GATE"; +const READY_ENV: &str = "A3S_RUNTIME_PROCESS_READY"; +const RESULT_ENV: &str = "A3S_RUNTIME_PROCESS_RESULT"; +const DRIVER_FAILPOINT_ENV: &str = "A3S_RUNTIME_PROCESS_DRIVER_FAILPOINT"; +const DRIVER_FAILPOINT_READY_ENV: &str = "A3S_RUNTIME_PROCESS_DRIVER_FAILPOINT_READY"; +const APPLY_AFTER_PUBLISH: &str = "provider.apply.after-current-publish"; + +fn service_spec(unit_id: &str, generation: u64) -> RuntimeUnitSpec { + RuntimeUnitSpec { + schema: RuntimeUnitSpec::SCHEMA.into(), + unit_id: unit_id.into(), + generation, + class: RuntimeUnitClass::Service, + artifact: ArtifactRef { + uri: format!( + "oci://registry.example/a3s/process-race@sha256:{}", + "a".repeat(64) + ), + digest: format!("sha256:{}", "a".repeat(64)), + media_type: IMAGE_MEDIA_TYPE.into(), + }, + process: RuntimeProcessSpec { + command: vec!["/bin/service".into()], + args: Vec::new(), + working_directory: None, + environment: BTreeMap::new(), + }, + mounts: Vec::new(), + secrets: Vec::new(), + network: RuntimeNetworkSpec { + mode: NetworkMode::None, + ports: Vec::new(), + }, + resources: ResourceLimits { + cpu_millis: 100, + memory_bytes: 64 * 1024 * 1024, + pids: 32, + ephemeral_storage_bytes: None, + execution_timeout_ms: None, + }, + isolation: IsolationLevel::Container, + health: None, + restart: RestartPolicy::Always, + outputs: Vec::new(), + semantics_profile_digest: None, + } +} + +fn apply_request(request_id: &str, unit_id: &str, generation: u64) -> RuntimeApplyRequest { + RuntimeApplyRequest { + schema: RuntimeApplyRequest::SCHEMA.into(), + request_id: request_id.into(), + deadline_at_ms: None, + spec: service_spec(unit_id, generation), + } +} + +fn action_request(request_id: &str, unit_id: &str, generation: u64) -> RuntimeActionRequest { + RuntimeActionRequest { + schema: RuntimeActionRequest::SCHEMA.into(), + request_id: request_id.into(), + unit_id: unit_id.into(), + generation, + deadline_at_ms: None, + } +} + +fn runtime_client(state_root: &Path, provider_root: &Path) -> ManagedRuntimeClient { + ManagedRuntimeClient::new( + Arc::new(FileRuntimeStateStore::new(state_root)), + Arc::new(ProcessRaceDriver::new(provider_root)), + ) +} + +fn test_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .expect("build process-race test runtime") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn subprocess_process_race_operation_helper() { + let Ok(state_root) = std::env::var(STATE_ROOT_ENV) else { + return; + }; + let provider_root = PathBuf::from(std::env::var(PROVIDER_ROOT_ENV).expect("provider root")); + let operation = std::env::var(OPERATION_ENV).expect("operation"); + let request_id = std::env::var(REQUEST_ID_ENV).expect("request ID"); + let unit_id = std::env::var(UNIT_ID_ENV).expect("unit ID"); + let generation = std::env::var(GENERATION_ENV) + .expect("generation") + .parse::() + .expect("numeric generation"); + let result = PathBuf::from(std::env::var(RESULT_ENV).expect("result path")); + + if let Ok(ready) = std::env::var(READY_ENV) { + std::fs::write(ready, b"ready").expect("publish process helper readiness"); + } + if let Ok(gate) = std::env::var(START_GATE_ENV) { + let gate = PathBuf::from(gate); + let deadline = Instant::now() + Duration::from_secs(10); + while !gate.is_file() { + assert!( + Instant::now() < deadline, + "process helper start gate timed out" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + let client = runtime_client(Path::new(&state_root), &provider_root); + let outcome = match operation.as_str() { + "apply" => client + .apply(&apply_request(&request_id, &unit_id, generation)) + .await + .map(|_| ()), + "stop" => client + .stop(&action_request(&request_id, &unit_id, generation)) + .await + .map(|_| ()), + "remove" => client + .remove(&action_request(&request_id, &unit_id, generation)) + .await + .map(|_| ()), + other => panic!("unknown process helper operation {other:?}"), + }; + std::fs::write(result, classify(outcome)).expect("write process helper result"); +} + +fn classify(result: Result<(), RuntimeError>) -> &'static str { + match result { + Ok(()) => "ok", + Err(RuntimeError::StaleGeneration { .. }) => "stale-generation", + Err(RuntimeError::GenerationConflict { .. }) => "generation-conflict", + Err(RuntimeError::NotFound { .. }) => "not-found", + Err(RuntimeError::Protocol(_)) => "protocol", + Err(_) => "unexpected-error", + } +} + +struct ProcessCase { + child: Child, + ready: PathBuf, + result: PathBuf, +} + +fn spawn_operation( + state_root: &Path, + provider_root: &Path, + gate: &Path, + operation: &str, + request_id: &str, + unit_id: &str, + generation: u64, +) -> ProcessCase { + let result = state_root.join(format!("{request_id}.result")); + let ready = state_root.join(format!("{request_id}.ready")); + let child = Command::new(std::env::current_exe().expect("current test executable")) + .arg("subprocess_process_race_operation_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env(STATE_ROOT_ENV, state_root) + .env(PROVIDER_ROOT_ENV, provider_root) + .env(OPERATION_ENV, operation) + .env(REQUEST_ID_ENV, request_id) + .env(UNIT_ID_ENV, unit_id) + .env(GENERATION_ENV, generation.to_string()) + .env(START_GATE_ENV, gate) + .env(READY_ENV, &ready) + .env(RESULT_ENV, &result) + .spawn() + .expect("spawn process race operation"); + ProcessCase { + child, + ready, + result, + } +} + +fn wait_for_ready(case: &mut ProcessCase, case_id: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + while !case.ready.is_file() { + if let Some(status) = case.child.try_wait().expect("inspect process helper") { + panic!("{case_id}: helper exited before ready: {status}"); + } + assert!(Instant::now() < deadline, "{case_id}: readiness timed out"); + thread::sleep(Duration::from_millis(10)); + } +} + +fn wait_for_exit(case: &mut ProcessCase, case_id: &str) -> String { + let deadline = Instant::now() + Duration::from_secs(20); + loop { + if let Some(status) = case.child.try_wait().expect("inspect process helper") { + assert!(status.success(), "{case_id}: helper failed: {status}"); + return std::fs::read_to_string(&case.result).expect("read process helper result"); + } + if Instant::now() >= deadline { + case.child.kill().expect("kill timed-out process helper"); + case.child + .wait() + .expect("wait for timed-out process helper"); + panic!("{case_id}: helper timed out"); + } + thread::sleep(Duration::from_millis(10)); + } +} + +fn run_pair( + state_root: &Path, + provider_root: &Path, + case_id: &str, + left: (&str, &str, &str, u64), + right: (&str, &str, &str, u64), +) -> (String, String) { + let gate = state_root.join(format!("{case_id}.gate")); + let mut left_case = spawn_operation( + state_root, + provider_root, + &gate, + left.0, + left.1, + left.2, + left.3, + ); + let mut right_case = spawn_operation( + state_root, + provider_root, + &gate, + right.0, + right.1, + right.2, + right.3, + ); + wait_for_ready(&mut left_case, case_id); + wait_for_ready(&mut right_case, case_id); + std::fs::write(&gate, b"start").expect("release process race gate"); + ( + wait_for_exit(&mut left_case, case_id), + wait_for_exit(&mut right_case, case_id), + ) +} + +fn assert_single_generation(resources: &[ProviderResource], generation: u64) { + assert_eq!(resources.len(), 1, "provider inventory is not singular"); + assert_eq!(resources[0].generation, generation); +} + +fn inject_duplicate(provider_root: &Path, unit_id: &str, generation: u64) { + let driver = ProcessRaceDriver::new(provider_root); + let source = driver + .inventory(unit_id) + .expect("load duplicate source") + .into_iter() + .find(|resource| resource.generation == generation) + .expect("duplicate source is absent"); + let mut duplicate = source; + duplicate.resource_id = format!("{}/duplicate", duplicate.resource_id); + let key = format!("{:x}", Sha256::digest(duplicate.resource_id.as_bytes())); + let path = provider_root.join("resources").join(format!("{key}.json")); + std::fs::write( + path, + serde_json::to_vec(&duplicate).expect("encode duplicate provider resource"), + ) + .expect("inject duplicate provider resource"); +} + +#[test] +fn race_gen_001_concurrent_generations_converge_to_one_newest_resource() { + let state = tempfile::tempdir().expect("generation-race state root"); + let provider = tempfile::tempdir().expect("generation-race provider root"); + let unit_id = "race-generation-unit"; + let (generation_one, generation_two) = run_pair( + state.path(), + provider.path(), + "RACE-GEN-001", + ("apply", "race-generation-one", unit_id, 1), + ("apply", "race-generation-two", unit_id, 2), + ); + assert!(matches!(generation_one.as_str(), "ok" | "stale-generation")); + assert_eq!(generation_two, "ok"); + + let runtime = test_runtime(); + let store = FileRuntimeStateStore::new(state.path()); + let record = runtime + .block_on(store.load(unit_id)) + .expect("load final unit"); + assert_eq!(record.spec.generation, 2); + assert_eq!(record.observation.state, RuntimeUnitState::Running); + assert_single_generation( + &ProcessRaceDriver::new(provider.path()) + .inventory(unit_id) + .expect("final provider inventory"), + 2, + ); +} + +#[test] +fn race_ops_001_apply_stop_remove_process_races_have_deterministic_oracles() { + for (suffix, competing_operation) in [("stop", "stop"), ("remove", "remove")] { + let state = tempfile::tempdir().expect("operation-race state root"); + let provider = tempfile::tempdir().expect("operation-race provider root"); + let unit_id = format!("race-apply-{suffix}"); + let runtime = test_runtime(); + runtime + .block_on( + runtime_client(state.path(), provider.path()).apply(&apply_request( + &format!("prepare-{suffix}"), + &unit_id, + 1, + )), + ) + .expect("prepare operation race"); + let (apply_result, action_result) = run_pair( + state.path(), + provider.path(), + &format!("RACE-OPS-001-apply-{suffix}"), + ("apply", &format!("apply-{suffix}-g2"), &unit_id, 2), + (competing_operation, &format!("{suffix}-g1"), &unit_id, 1), + ); + assert_eq!(apply_result, "ok"); + assert!(matches!( + action_result.as_str(), + "ok" | "stale-generation" | "not-found" + )); + let record = runtime + .block_on(FileRuntimeStateStore::new(state.path()).load(&unit_id)) + .expect("load apply/action race unit"); + assert_eq!(record.spec.generation, 2); + assert_eq!(record.observation.state, RuntimeUnitState::Running); + assert_single_generation( + &ProcessRaceDriver::new(provider.path()) + .inventory(&unit_id) + .expect("apply/action provider inventory"), + 2, + ); + } + + let state = tempfile::tempdir().expect("stop-remove state root"); + let provider = tempfile::tempdir().expect("stop-remove provider root"); + let unit_id = "race-stop-remove"; + let runtime = test_runtime(); + runtime + .block_on( + runtime_client(state.path(), provider.path()).apply(&apply_request( + "prepare-stop-remove", + unit_id, + 1, + )), + ) + .expect("prepare stop/remove race"); + let (stop_result, remove_result) = run_pair( + state.path(), + provider.path(), + "RACE-OPS-001-stop-remove", + ("stop", "race-stop", unit_id, 1), + ("remove", "race-remove", unit_id, 1), + ); + assert!(matches!(stop_result.as_str(), "ok" | "not-found")); + assert_eq!(remove_result, "ok"); + let record = runtime + .block_on(FileRuntimeStateStore::new(state.path()).load(unit_id)) + .expect("load stop/remove race unit"); + assert!(record.removed_at_ms.is_some()); + assert!(ProcessRaceDriver::new(provider.path()) + .inventory(unit_id) + .expect("stop/remove provider inventory") + .is_empty()); +} + +#[test] +fn crash_gen_001_killed_generation_handoff_exact_replay_converges_once() { + let state = tempfile::tempdir().expect("generation-crash state root"); + let provider = tempfile::tempdir().expect("generation-crash provider root"); + let unit_id = "generation-crash-unit"; + let runtime = test_runtime(); + runtime + .block_on( + runtime_client(state.path(), provider.path()).apply(&apply_request( + "generation-crash-one", + unit_id, + 1, + )), + ) + .expect("prepare generation crash"); + + let result = state.path().join("generation-crash-two.result"); + let failpoint_ready = state.path().join("generation-crash-two.failpoint"); + let mut child = Command::new(std::env::current_exe().expect("current test executable")) + .arg("subprocess_process_race_operation_helper") + .arg("--nocapture") + .arg("--test-threads=1") + .env(STATE_ROOT_ENV, state.path()) + .env(PROVIDER_ROOT_ENV, provider.path()) + .env(OPERATION_ENV, "apply") + .env(REQUEST_ID_ENV, "generation-crash-two") + .env(UNIT_ID_ENV, unit_id) + .env(GENERATION_ENV, "2") + .env(RESULT_ENV, &result) + .env(DRIVER_FAILPOINT_ENV, APPLY_AFTER_PUBLISH) + .env(DRIVER_FAILPOINT_READY_ENV, &failpoint_ready) + .spawn() + .expect("spawn generation crash helper"); + let deadline = Instant::now() + Duration::from_secs(10); + while !failpoint_ready.is_file() { + if let Some(status) = child.try_wait().expect("inspect generation crash helper") { + panic!("generation crash helper exited early: {status}"); + } + assert!(Instant::now() < deadline, "generation failpoint timed out"); + thread::sleep(Duration::from_millis(10)); + } + child.kill().expect("kill generation crash helper"); + assert!(!child + .wait() + .expect("wait generation crash helper") + .success()); + + let store = FileRuntimeStateStore::new(state.path()); + let interrupted = runtime + .block_on(store.load(unit_id)) + .expect("load interrupted unit"); + assert_eq!(interrupted.spec.generation, 2); + assert_eq!(interrupted.observation.state, RuntimeUnitState::Accepted); + assert_eq!( + runtime + .block_on(store.load_request(unit_id, "generation-crash-two")) + .expect("load interrupted receipt") + .state, + RuntimeRequestState::Pending + ); + let process_driver = ProcessRaceDriver::new(provider.path()); + let interrupted_inventory = process_driver + .inventory(unit_id) + .expect("interrupted provider inventory"); + assert_eq!( + interrupted_inventory + .iter() + .map(|resource| resource.generation) + .collect::>(), + vec![1, 2] + ); + + let request = apply_request("generation-crash-two", unit_id, 2); + let recovered = runtime + .block_on(runtime_client(state.path(), provider.path()).apply(&request)) + .expect("recover generation handoff"); + assert_eq!(recovered.state, RuntimeUnitState::Running); + let converged = process_driver + .inventory(unit_id) + .expect("converged provider inventory"); + assert_single_generation(&converged, 2); + assert_eq!(converged[0].apply_dispatches, 2); + let replayed = runtime + .block_on(runtime_client(state.path(), provider.path()).apply(&request)) + .expect("replay recovered generation"); + assert_eq!(replayed, recovered); + assert_eq!( + process_driver + .inventory(unit_id) + .expect("replayed provider inventory"), + converged + ); +} + +#[test] +fn race_inventory_001_duplicate_resources_fail_closed_before_provider_mutation() { + let state = tempfile::tempdir().expect("duplicate state root"); + let provider = tempfile::tempdir().expect("duplicate provider root"); + let unit_id = "duplicate-provider-unit"; + let runtime = test_runtime(); + let client = runtime_client(state.path(), provider.path()); + runtime + .block_on(client.apply(&apply_request("duplicate-one", unit_id, 1))) + .expect("prepare duplicate provider fixture"); + let process_driver = ProcessRaceDriver::new(provider.path()); + inject_duplicate(provider.path(), unit_id, 1); + assert_eq!( + process_driver + .inventory(unit_id) + .expect("duplicate provider inventory") + .len(), + 2 + ); + assert!(matches!( + runtime.block_on(client.inspect(unit_id)), + Err(RuntimeError::Protocol(message)) if message.contains("duplicate resources") + )); + assert!(matches!( + runtime.block_on(client.apply(&apply_request("duplicate-two", unit_id, 2))), + Err(RuntimeError::Protocol(message)) if message.contains("duplicate resources") + )); + let resources = process_driver + .inventory(unit_id) + .expect("post-rejection provider inventory"); + assert_eq!(resources.len(), 2); + assert!(resources.iter().all(|resource| resource.generation == 1)); + let record = runtime + .block_on(FileRuntimeStateStore::new(state.path()).load(unit_id)) + .expect("load duplicate-rejection state"); + assert_eq!(record.spec.generation, 2); + assert_eq!(record.observation.state, RuntimeUnitState::Accepted); +} diff --git a/tests/process_races/driver.rs b/tests/process_races/driver.rs new file mode 100644 index 0000000..23bc47c --- /dev/null +++ b/tests/process_races/driver.rs @@ -0,0 +1,468 @@ +use a3s_runtime::contract::{ + IsolationLevel, NetworkMode, ResourceControl, RuntimeActionRequest, RuntimeCapabilities, + RuntimeExecRequest, RuntimeExecResult, RuntimeFailure, RuntimeFeature, RuntimeInspection, + RuntimeLogChunk, RuntimeLogQuery, RuntimeObservation, RuntimeRemoval, RuntimeUnitClass, + RuntimeUnitSpec, RuntimeUnitState, +}; +use a3s_runtime::{ProviderId, RuntimeDriver, RuntimeError, RuntimeResult, RuntimeUnitRecord}; +use async_trait::async_trait; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub const IMAGE_MEDIA_TYPE: &str = "application/vnd.oci.image.manifest.v1+json"; +const PROVIDER_BUILD: &str = "process-race-driver/1"; +const FAILPOINT_ENV: &str = "A3S_RUNTIME_PROCESS_DRIVER_FAILPOINT"; +const FAILPOINT_READY_ENV: &str = "A3S_RUNTIME_PROCESS_DRIVER_FAILPOINT_READY"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProviderResource { + pub resource_id: String, + pub unit_id: String, + pub generation: u64, + pub state: RuntimeUnitState, + pub observed_at_ms: u64, + pub started_at_ms: u64, + pub apply_dispatches: u64, +} + +#[derive(Debug, Clone)] +pub struct ProcessRaceDriver { + root: PathBuf, + provider_id: ProviderId, +} + +impl ProcessRaceDriver { + pub fn new(root: impl Into) -> Self { + Self { + root: root.into(), + provider_id: ProviderId::parse("process-race-runtime") + .expect("valid process-race provider ID"), + } + } + + pub fn inventory(&self, unit_id: &str) -> RuntimeResult> { + self.with_lock(|root| inventory_unlocked(root, unit_id)) + } + + fn with_lock(&self, operation: impl FnOnce(&Path) -> RuntimeResult) -> RuntimeResult { + ensure_provider_root(&self.root)?; + let lock_path = self.root.join("provider.lock"); + let mut options = OpenOptions::new(); + options.create(true).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let lock = options + .open(lock_path) + .map_err(io_error("open process provider lock"))?; + lock.lock_exclusive() + .map_err(io_error("lock process provider"))?; + let _guard = ProviderLock(lock); + operation(&self.root) + } + + fn apply_sync( + &self, + spec: &RuntimeUnitSpec, + current: &RuntimeObservation, + ) -> RuntimeResult { + self.with_lock(|root| { + let resources = inventory_unlocked(root, &spec.unit_id)?; + if let Some(generation) = duplicate_generation(&resources) { + return Err(duplicate_error(&spec.unit_id, generation)); + } + let current_resources = resources + .iter() + .filter(|resource| resource.generation == spec.generation) + .collect::>(); + + let target_state = match spec.class { + RuntimeUnitClass::Task + if spec + .process + .args + .iter() + .any(|argument| matches!(argument.as_str(), "fail" | "timeout")) => + { + RuntimeUnitState::Failed + } + RuntimeUnitClass::Task => RuntimeUnitState::Succeeded, + RuntimeUnitClass::Service => RuntimeUnitState::Running, + }; + let mut resource = current_resources + .first() + .copied() + .cloned() + .unwrap_or_else(|| ProviderResource { + resource_id: format!("process/{}/g{}", spec.unit_id, spec.generation), + unit_id: spec.unit_id.clone(), + generation: spec.generation, + state: target_state, + observed_at_ms: current.observed_at_ms.saturating_add(1), + started_at_ms: current.observed_at_ms.saturating_add(1), + apply_dispatches: 0, + }); + resource.apply_dispatches = resource.apply_dispatches.saturating_add(1); + resource.state = target_state; + resource.observed_at_ms = resource + .observed_at_ms + .max(current.observed_at_ms.saturating_add(1)); + write_resource(root, &resource)?; + hit_failpoint("provider.apply.after-current-publish"); + + for stale in resources + .iter() + .filter(|candidate| candidate.generation != spec.generation) + { + remove_resource(root, &stale.resource_id)?; + } + let final_inventory = inventory_unlocked(root, &spec.unit_id)?; + if final_inventory.len() != 1 + || final_inventory[0].generation != spec.generation + || final_inventory[0].resource_id != resource.resource_id + { + return Err(RuntimeError::Protocol(format!( + "process provider did not converge {:?} generation {} to one resource", + spec.unit_id, spec.generation + ))); + } + Ok(observation_from_resource(current, &resource)) + }) + } + + fn inspect_sync(&self, unit: &RuntimeUnitRecord) -> RuntimeResult { + self.with_lock(|root| { + let resources = inventory_unlocked(root, &unit.spec.unit_id)?; + let current = resources + .iter() + .filter(|resource| resource.generation == unit.spec.generation) + .collect::>(); + if current.len() > 1 { + return Err(duplicate_error(&unit.spec.unit_id, unit.spec.generation)); + } + let Some(resource) = current.first() else { + return Ok(RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), + unit_id: unit.spec.unit_id.clone(), + last_generation: Some(unit.spec.generation), + }); + }; + Ok(RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), + observation: Box::new(observation_from_resource(&unit.observation, resource)), + }) + }) + } + + fn stop_sync(&self, unit: &RuntimeUnitRecord) -> RuntimeResult { + self.with_lock(|root| { + let resources = inventory_unlocked(root, &unit.spec.unit_id)?; + let mut current = resources + .into_iter() + .filter(|resource| resource.generation == unit.spec.generation) + .collect::>(); + if current.len() > 1 { + return Err(duplicate_error(&unit.spec.unit_id, unit.spec.generation)); + } + let Some(mut resource) = current.pop() else { + let mut unknown = unit.observation.clone(); + unknown.state = RuntimeUnitState::Unknown; + unknown.observed_at_ms = unknown.observed_at_ms.saturating_add(1); + unknown.finished_at_ms = None; + unknown.health = None; + unknown.outputs.clear(); + unknown.failure = None; + return Ok(unknown); + }; + resource.state = RuntimeUnitState::Stopped; + resource.observed_at_ms = resource + .observed_at_ms + .max(unit.observation.observed_at_ms.saturating_add(1)); + write_resource(root, &resource)?; + Ok(observation_from_resource(&unit.observation, &resource)) + }) + } + + fn remove_sync( + &self, + unit: &RuntimeUnitRecord, + request: &RuntimeActionRequest, + ) -> RuntimeResult { + self.with_lock(|root| { + let resources = inventory_unlocked(root, &unit.spec.unit_id)?; + let current = resources + .iter() + .filter(|resource| resource.generation == unit.spec.generation) + .collect::>(); + if current.len() > 1 { + return Err(duplicate_error(&unit.spec.unit_id, unit.spec.generation)); + } + let already_absent = current.is_empty(); + if let Some(resource) = current.first() { + remove_resource(root, &resource.resource_id)?; + } + Ok(RuntimeRemoval { + schema: RuntimeRemoval::SCHEMA.into(), + request_id: request.request_id.clone(), + unit_id: request.unit_id.clone(), + generation: request.generation, + removed_at_ms: now_ms(), + already_absent, + }) + }) + } +} + +#[async_trait] +impl RuntimeDriver for ProcessRaceDriver { + fn provider_id(&self) -> &ProviderId { + &self.provider_id + } + + async fn capabilities(&self) -> RuntimeResult { + Ok(RuntimeCapabilities { + schema: RuntimeCapabilities::SCHEMA.into(), + provider_id: self.provider_id.clone(), + provider_build: PROVIDER_BUILD.into(), + unit_classes: vec![RuntimeUnitClass::Task, RuntimeUnitClass::Service], + artifact_media_types: vec![IMAGE_MEDIA_TYPE.into()], + isolation_levels: vec![IsolationLevel::Container], + network_modes: vec![NetworkMode::None], + mount_kinds: Vec::new(), + health_check_kinds: Vec::new(), + resource_controls: vec![ + ResourceControl::Cpu, + ResourceControl::Memory, + ResourceControl::Pids, + ResourceControl::ExecutionTimeout, + ], + features: vec![ + RuntimeFeature::DurableIdentity, + RuntimeFeature::Stop, + RuntimeFeature::Remove, + ], + }) + } + + async fn apply( + &self, + spec: &RuntimeUnitSpec, + current: &RuntimeObservation, + ) -> RuntimeResult { + let driver = self.clone(); + let spec = spec.clone(); + let current = current.clone(); + tokio::task::spawn_blocking(move || driver.apply_sync(&spec, ¤t)) + .await + .map_err(task_error)? + } + + async fn inspect(&self, unit: &RuntimeUnitRecord) -> RuntimeResult { + let driver = self.clone(); + let unit = unit.clone(); + tokio::task::spawn_blocking(move || driver.inspect_sync(&unit)) + .await + .map_err(task_error)? + } + + async fn stop( + &self, + unit: &RuntimeUnitRecord, + _request: &RuntimeActionRequest, + ) -> RuntimeResult { + let driver = self.clone(); + let unit = unit.clone(); + tokio::task::spawn_blocking(move || driver.stop_sync(&unit)) + .await + .map_err(task_error)? + } + + async fn remove( + &self, + unit: &RuntimeUnitRecord, + request: &RuntimeActionRequest, + ) -> RuntimeResult { + let driver = self.clone(); + let unit = unit.clone(); + let request = request.clone(); + tokio::task::spawn_blocking(move || driver.remove_sync(&unit, &request)) + .await + .map_err(task_error)? + } + + async fn logs( + &self, + _unit: &RuntimeUnitRecord, + _query: &RuntimeLogQuery, + ) -> RuntimeResult> { + Err(RuntimeError::UnsupportedCapabilities(vec![ + "feature:Logs".into() + ])) + } + + async fn exec( + &self, + _unit: &RuntimeUnitRecord, + _request: &RuntimeExecRequest, + ) -> RuntimeResult { + Err(RuntimeError::UnsupportedCapabilities(vec![ + "feature:Exec".into() + ])) + } +} + +fn ensure_provider_root(root: &Path) -> RuntimeResult<()> { + std::fs::create_dir_all(root.join("resources")) + .map_err(io_error("create process provider root"))?; + Ok(()) +} + +fn inventory_unlocked(root: &Path, unit_id: &str) -> RuntimeResult> { + let mut resources = Vec::new(); + for entry in std::fs::read_dir(root.join("resources")) + .map_err(io_error("read process provider inventory"))? + { + let entry = entry.map_err(io_error("read process provider resource"))?; + if entry.path().extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let bytes = std::fs::read(entry.path()).map_err(io_error("read provider resource"))?; + let resource: ProviderResource = serde_json::from_slice(&bytes).map_err(|error| { + RuntimeError::Protocol(format!("invalid provider resource: {error}")) + })?; + if resource.unit_id == unit_id { + resources.push(resource); + } + } + resources.sort_by(|left, right| { + (left.generation, &left.resource_id).cmp(&(right.generation, &right.resource_id)) + }); + Ok(resources) +} + +fn write_resource(root: &Path, resource: &ProviderResource) -> RuntimeResult<()> { + let path = resource_path(root, &resource.resource_id); + let temporary = path.with_extension(format!("{}.tmp", std::process::id())); + let bytes = serde_json::to_vec(resource) + .map_err(|error| RuntimeError::Protocol(format!("encode provider resource: {error}")))?; + let mut options = OpenOptions::new(); + options.create(true).truncate(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(&temporary) + .map_err(io_error("create provider staging file"))?; + file.write_all(&bytes) + .and_then(|()| file.sync_all()) + .map_err(io_error("write provider resource"))?; + std::fs::rename(&temporary, &path).map_err(io_error("publish provider resource"))?; + sync_directory(&root.join("resources")) +} + +fn remove_resource(root: &Path, resource_id: &str) -> RuntimeResult<()> { + let path = resource_path(root, resource_id); + match std::fs::remove_file(path) { + Ok(()) => sync_directory(&root.join("resources")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(io_error("remove provider resource")(error)), + } +} + +fn resource_path(root: &Path, resource_id: &str) -> PathBuf { + let key = format!("{:x}", Sha256::digest(resource_id.as_bytes())); + root.join("resources").join(format!("{key}.json")) +} + +fn observation_from_resource( + current: &RuntimeObservation, + resource: &ProviderResource, +) -> RuntimeObservation { + let mut observation = current.clone(); + observation.state = resource.state; + observation.provider_resource_id = Some(resource.resource_id.clone()); + observation.provider_build = Some(PROVIDER_BUILD.into()); + observation.observed_at_ms = resource.observed_at_ms.max(current.observed_at_ms); + observation.started_at_ms = Some(resource.started_at_ms); + observation.finished_at_ms = resource + .state + .is_terminal() + .then_some(observation.observed_at_ms); + observation.health = None; + observation.outputs.clear(); + observation.failure = (resource.state == RuntimeUnitState::Failed).then(|| RuntimeFailure { + code: "fixture_failure".into(), + message: "process-race fixture failed as requested".into(), + retryable: false, + }); + observation +} + +fn duplicate_error(unit_id: &str, generation: u64) -> RuntimeError { + RuntimeError::Protocol(format!( + "process provider found duplicate resources for {unit_id:?} generation {generation}" + )) +} + +fn duplicate_generation(resources: &[ProviderResource]) -> Option { + resources + .windows(2) + .find_map(|pair| (pair[0].generation == pair[1].generation).then_some(pair[0].generation)) +} + +fn hit_failpoint(name: &str) { + if !matches!(std::env::var(FAILPOINT_ENV), Ok(value) if value == name) { + return; + } + let ready = std::env::var(FAILPOINT_READY_ENV).expect("provider failpoint ready path"); + std::fs::write(ready, name).expect("publish provider failpoint readiness"); + loop { + std::thread::park_timeout(std::time::Duration::from_secs(60)); + } +} + +fn now_ms() -> u64 { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + u64::try_from(millis).unwrap_or(u64::MAX) +} + +#[cfg(unix)] +fn sync_directory(path: &Path) -> RuntimeResult<()> { + File::open(path) + .and_then(|directory| directory.sync_all()) + .map_err(io_error("sync process provider directory")) +} + +#[cfg(not(unix))] +fn sync_directory(_path: &Path) -> RuntimeResult<()> { + Ok(()) +} + +fn task_error(error: tokio::task::JoinError) -> RuntimeError { + RuntimeError::Transport(format!("process provider task failed: {error}")) +} + +fn io_error(action: &'static str) -> impl FnOnce(std::io::Error) -> RuntimeError { + move |error| RuntimeError::Transport(format!("could not {action}: {error}")) +} + +struct ProviderLock(File); + +impl Drop for ProviderLock { + fn drop(&mut self) { + let _ = FileExt::unlock(&self.0); + } +} diff --git a/tests/protocol_golden.rs b/tests/protocol_golden.rs new file mode 100644 index 0000000..f1140bc --- /dev/null +++ b/tests/protocol_golden.rs @@ -0,0 +1,185 @@ +use a3s_runtime::contract::{ + RuntimeActionRequest, RuntimeApplyRequest, RuntimeCapabilities, RuntimeExecRequest, + RuntimeExecResult, RuntimeInspection, RuntimeLogChunk, RuntimeLogQuery, RuntimeObservation, + RuntimeRemoval, RuntimeUnitSpec, +}; +use a3s_runtime::{RuntimeRequestReceipt, RuntimeUnitRecord}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use serde_json::{Map, Value}; + +fn adjacent_schema(schema: &str, offset: i64) -> String { + let (prefix, version) = schema + .rsplit_once(".v") + .expect("versioned schema must end in .v"); + let version = version + .parse::() + .expect("schema version must be numeric"); + format!("{prefix}.v{}", version + offset) +} + +fn object(value: &mut Value) -> &mut Map { + value + .as_object_mut() + .expect("every top-level Runtime wire fixture must be an object") +} + +fn assert_top_level_fixture( + raw: &str, + expected_schema: &str, + validate: impl Fn(&T) -> Result<(), String>, +) where + T: DeserializeOwned + Serialize, +{ + let value: Value = serde_json::from_str(raw).expect("golden fixture must be valid JSON"); + assert_eq!( + value.get("schema"), + Some(&Value::String(expected_schema.into())) + ); + + let decoded: T = + serde_json::from_value(value.clone()).expect("current golden fixture must decode"); + validate(&decoded).expect("current golden fixture must validate"); + assert_eq!( + serde_json::to_value(&decoded).expect("golden value must encode"), + value, + "golden encode must preserve the complete public wire shape" + ); + + let mut missing = value.clone(); + object(&mut missing).remove("schema"); + assert!( + serde_json::from_value::(missing).is_err(), + "a missing schema must fail closed" + ); + + for incompatible_schema in [ + adjacent_schema(expected_schema, -1), + adjacent_schema(expected_schema, 1), + ] { + let mut incompatible = value.clone(); + object(&mut incompatible).insert("schema".into(), incompatible_schema.into()); + let decoded: T = serde_json::from_value(incompatible) + .expect("a syntactically valid schema string must decode before validation"); + assert!( + validate(&decoded).is_err(), + "old and future schemas must fail validation" + ); + } + + let mut malformed = value.clone(); + object(&mut malformed).insert("schema".into(), Value::Bool(true)); + assert!( + serde_json::from_value::(malformed).is_err(), + "a non-string schema must fail decoding" + ); + + let mut unknown = value; + object(&mut unknown).insert("unexpected".into(), Value::Bool(true)); + assert!( + serde_json::from_value::(unknown).is_err(), + "unknown top-level fields must fail closed" + ); +} + +#[test] +fn ct_schema_001_every_top_level_wire_record_has_a_versioned_golden_fixture() { + assert_top_level_fixture::( + include_str!("golden/capabilities-v3.json"), + RuntimeCapabilities::SCHEMA, + RuntimeCapabilities::validate, + ); + assert_top_level_fixture::( + include_str!("golden/unit-spec-v2.json"), + RuntimeUnitSpec::SCHEMA, + RuntimeUnitSpec::validate, + ); + assert_top_level_fixture::( + include_str!("golden/apply-request-v1.json"), + RuntimeApplyRequest::SCHEMA, + RuntimeApplyRequest::validate, + ); + assert_top_level_fixture::( + include_str!("golden/action-request-v1.json"), + RuntimeActionRequest::SCHEMA, + RuntimeActionRequest::validate, + ); + assert_top_level_fixture::( + include_str!("golden/observation-v2.json"), + RuntimeObservation::SCHEMA, + RuntimeObservation::validate, + ); + for inspection in [ + include_str!("golden/inspection-found-v1.json"), + include_str!("golden/inspection-not-found-v1.json"), + ] { + assert_top_level_fixture::( + inspection, + RuntimeInspection::SCHEMA, + RuntimeInspection::validate, + ); + } + assert_top_level_fixture::( + include_str!("golden/removal-v1.json"), + RuntimeRemoval::SCHEMA, + RuntimeRemoval::validate, + ); + assert_top_level_fixture::( + include_str!("golden/log-query-v1.json"), + RuntimeLogQuery::SCHEMA, + RuntimeLogQuery::validate, + ); + assert_top_level_fixture::( + include_str!("golden/log-chunk-v1.json"), + RuntimeLogChunk::SCHEMA, + RuntimeLogChunk::validate, + ); + assert_top_level_fixture::( + include_str!("golden/exec-request-v1.json"), + RuntimeExecRequest::SCHEMA, + RuntimeExecRequest::validate, + ); + assert_top_level_fixture::( + include_str!("golden/exec-result-v1.json"), + RuntimeExecResult::SCHEMA, + RuntimeExecResult::validate, + ); + assert_top_level_fixture::( + include_str!("golden/unit-record-v2.json"), + RuntimeUnitRecord::SCHEMA, + RuntimeUnitRecord::validate, + ); + assert_top_level_fixture::( + include_str!("golden/request-receipt-v2.json"), + RuntimeRequestReceipt::SCHEMA, + RuntimeRequestReceipt::validate, + ); +} + +#[test] +fn ct_digest_001_golden_request_and_spec_digests_are_stable() { + let unit: RuntimeUnitSpec = + serde_json::from_str(include_str!("golden/unit-spec-v2.json")).unwrap(); + assert_eq!( + unit.digest().unwrap(), + "sha256:56d815af40b2bcb083a0fd8b42bcf4501330362d81b3850a45d901725d828610" + ); + + let apply: RuntimeApplyRequest = + serde_json::from_str(include_str!("golden/apply-request-v1.json")).unwrap(); + assert_eq!( + apply.spec.digest().unwrap(), + "sha256:3309c80ee072c47311be878a7fcde9818e484e9d186fa1c6eb78033fe5ef4d11" + ); + assert_eq!( + apply.digest().unwrap(), + "sha256:e178a719503c68db140756ae8568c3e51da35843de1133c3a6585520f0f4ea91" + ); + + let exec: RuntimeExecRequest = + serde_json::from_str(include_str!("golden/exec-request-v1.json")).unwrap(); + assert_eq!( + exec.digest().unwrap(), + "sha256:861f6cad44261c49a89b5cd12920ca3977d833b5c9143d748db945a08939bad0" + ); +} diff --git a/tests/support/fault_driver.rs b/tests/support/fault_driver.rs new file mode 100644 index 0000000..431b3a5 --- /dev/null +++ b/tests/support/fault_driver.rs @@ -0,0 +1,434 @@ +use a3s_runtime::contract::{ + IsolationLevel, NetworkMode, ResourceControl, RuntimeActionRequest, RuntimeCapabilities, + RuntimeExecRequest, RuntimeExecResult, RuntimeFeature, RuntimeInspection, RuntimeLogChunk, + RuntimeLogQuery, RuntimeLogStream, RuntimeObservation, RuntimeRemoval, RuntimeUnitClass, + RuntimeUnitSpec, RuntimeUnitState, +}; +use a3s_runtime::{ProviderId, RuntimeDriver, RuntimeError, RuntimeResult, RuntimeUnitRecord}; +use async_trait::async_trait; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Mutex; + +pub const IMAGE_MEDIA_TYPE: &str = "application/vnd.oci.image.manifest.v1+json"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum FaultOperation { + Capabilities, + Apply, + Inspect, + Stop, + Remove, + Logs, + Exec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FaultBoundary { + BeforeEffect, + AfterEffect, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FaultMode { + Transport, + Hang, + Panic, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ArmedFault { + operation: FaultOperation, + boundary: FaultBoundary, + mode: FaultMode, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TraceEvent { + pub operation: FaultOperation, + pub boundary: FaultBoundary, +} + +#[derive(Debug, Clone)] +struct ProviderResource { + observation: RuntimeObservation, +} + +pub struct DeterministicFaultDriver { + provider_id: ProviderId, + fault: Mutex>, + trace: Mutex>, + resources: Mutex>>, + exec_results: Mutex>, + effects: Mutex>, +} + +impl Default for DeterministicFaultDriver { + fn default() -> Self { + Self::new() + } +} + +impl DeterministicFaultDriver { + pub fn new() -> Self { + Self { + provider_id: ProviderId::parse("deterministic-fault-runtime") + .expect("valid deterministic provider ID"), + fault: Mutex::new(None), + trace: Mutex::new(Vec::new()), + resources: Mutex::new(BTreeMap::new()), + exec_results: Mutex::new(BTreeMap::new()), + effects: Mutex::new(BTreeMap::new()), + } + } + + pub fn arm(&self, operation: FaultOperation, boundary: FaultBoundary, mode: FaultMode) { + let mut fault = self.fault.lock().expect("fault plan lock"); + assert!(fault.is_none(), "only one deterministic fault may be armed"); + *fault = Some(ArmedFault { + operation, + boundary, + mode, + }); + } + + pub fn trace(&self) -> Vec { + self.trace.lock().expect("trace lock").clone() + } + + pub fn resource_count(&self, unit_id: &str) -> usize { + self.resources + .lock() + .expect("provider resource lock") + .get(unit_id) + .map(BTreeMap::len) + .unwrap_or_default() + } + + pub fn resource_state(&self, unit_id: &str, generation: u64) -> Option { + self.resources + .lock() + .expect("provider resource lock") + .get(unit_id) + .and_then(|generations| generations.get(&generation)) + .map(|resource| resource.observation.state) + } + + pub fn effect_count(&self, operation: FaultOperation) -> usize { + self.effects + .lock() + .expect("effect counter lock") + .get(&operation) + .copied() + .unwrap_or_default() + } + + async fn boundary( + &self, + operation: FaultOperation, + boundary: FaultBoundary, + ) -> RuntimeResult<()> { + self.trace.lock().expect("trace lock").push(TraceEvent { + operation, + boundary, + }); + let mode = { + let mut armed = self.fault.lock().expect("fault plan lock"); + if armed + .as_ref() + .is_some_and(|fault| fault.operation == operation && fault.boundary == boundary) + { + armed.take().map(|fault| fault.mode) + } else { + None + } + }; + match mode { + None => Ok(()), + Some(FaultMode::Transport) => Err(RuntimeError::Transport(format!( + "injected {operation:?} {boundary:?} transport fault" + ))), + Some(FaultMode::Hang) => std::future::pending::>().await, + Some(FaultMode::Panic) => { + panic!("injected {operation:?} {boundary:?} provider panic") + } + } + } + + fn record_effect(&self, operation: FaultOperation) { + *self + .effects + .lock() + .expect("effect counter lock") + .entry(operation) + .or_default() += 1; + } + + fn capabilities_value(&self) -> RuntimeCapabilities { + RuntimeCapabilities { + schema: RuntimeCapabilities::SCHEMA.into(), + provider_id: self.provider_id.clone(), + provider_build: "deterministic-fault-driver/1".into(), + unit_classes: vec![RuntimeUnitClass::Task, RuntimeUnitClass::Service], + artifact_media_types: vec![IMAGE_MEDIA_TYPE.into()], + isolation_levels: vec![IsolationLevel::Container], + network_modes: vec![NetworkMode::None], + mount_kinds: Vec::new(), + health_check_kinds: Vec::new(), + resource_controls: vec![ + ResourceControl::Cpu, + ResourceControl::Memory, + ResourceControl::Pids, + ResourceControl::ExecutionTimeout, + ], + features: vec![ + RuntimeFeature::DurableIdentity, + RuntimeFeature::Stop, + RuntimeFeature::Remove, + RuntimeFeature::Logs, + RuntimeFeature::Exec, + ], + } + } + + fn provider_observation( + &self, + spec: &RuntimeUnitSpec, + current: &RuntimeObservation, + resource_id: String, + ) -> RuntimeObservation { + let mut observation = current.clone(); + observation.state = match spec.class { + RuntimeUnitClass::Task => RuntimeUnitState::Succeeded, + RuntimeUnitClass::Service => RuntimeUnitState::Running, + }; + observation.provider_resource_id = Some(resource_id); + observation.provider_build = Some("deterministic-fault-driver/1".into()); + observation.observed_at_ms = current.observed_at_ms.saturating_add(1); + observation.started_at_ms = Some(current.observed_at_ms); + observation.finished_at_ms = observation + .state + .is_terminal() + .then_some(observation.observed_at_ms); + observation.health = None; + observation.outputs.clear(); + observation.failure = None; + observation + } + + fn exec_key(request: &RuntimeExecRequest) -> String { + format!( + "{}/{}/{}", + request.unit_id, request.generation, request.request_id + ) + } +} + +#[async_trait] +impl RuntimeDriver for DeterministicFaultDriver { + fn provider_id(&self) -> &ProviderId { + &self.provider_id + } + + async fn capabilities(&self) -> RuntimeResult { + self.boundary(FaultOperation::Capabilities, FaultBoundary::BeforeEffect) + .await?; + let capabilities = self.capabilities_value(); + self.boundary(FaultOperation::Capabilities, FaultBoundary::AfterEffect) + .await?; + Ok(capabilities) + } + + async fn apply( + &self, + spec: &RuntimeUnitSpec, + current: &RuntimeObservation, + ) -> RuntimeResult { + self.boundary(FaultOperation::Apply, FaultBoundary::BeforeEffect) + .await?; + let (observation, created) = { + let mut resources = self.resources.lock().expect("provider resource lock"); + let generations = resources.entry(spec.unit_id.clone()).or_default(); + let mut created = false; + let observation = if let Some(resource) = generations.get(&spec.generation) { + resource.observation.clone() + } else { + created = true; + self.provider_observation( + spec, + current, + format!("provider/{}/g{}", spec.unit_id, spec.generation), + ) + }; + generations.insert( + spec.generation, + ProviderResource { + observation: observation.clone(), + }, + ); + let stale = generations + .keys() + .copied() + .filter(|generation| *generation != spec.generation) + .collect::>(); + for generation in stale { + generations.remove(&generation); + } + (observation, created) + }; + if created { + self.record_effect(FaultOperation::Apply); + } + self.boundary(FaultOperation::Apply, FaultBoundary::AfterEffect) + .await?; + Ok(observation) + } + + async fn inspect(&self, unit: &RuntimeUnitRecord) -> RuntimeResult { + self.boundary(FaultOperation::Inspect, FaultBoundary::BeforeEffect) + .await?; + let inspection = self + .resources + .lock() + .expect("provider resource lock") + .get(&unit.spec.unit_id) + .and_then(|generations| generations.get(&unit.spec.generation)) + .map_or_else( + || RuntimeInspection::NotFound { + schema: RuntimeInspection::SCHEMA.into(), + unit_id: unit.spec.unit_id.clone(), + last_generation: Some(unit.spec.generation), + }, + |resource| RuntimeInspection::Found { + schema: RuntimeInspection::SCHEMA.into(), + observation: Box::new(resource.observation.clone()), + }, + ); + self.boundary(FaultOperation::Inspect, FaultBoundary::AfterEffect) + .await?; + Ok(inspection) + } + + async fn stop( + &self, + unit: &RuntimeUnitRecord, + _request: &RuntimeActionRequest, + ) -> RuntimeResult { + self.boundary(FaultOperation::Stop, FaultBoundary::BeforeEffect) + .await?; + let (observation, changed) = { + let mut resources = self.resources.lock().expect("provider resource lock"); + let resource = resources + .get_mut(&unit.spec.unit_id) + .and_then(|generations| generations.get_mut(&unit.spec.generation)) + .ok_or_else(|| RuntimeError::NotFound { + unit_id: unit.spec.unit_id.clone(), + })?; + let changed = resource.observation.state != RuntimeUnitState::Stopped; + if changed { + resource.observation.state = RuntimeUnitState::Stopped; + resource.observation.observed_at_ms = + resource.observation.observed_at_ms.saturating_add(1); + resource.observation.finished_at_ms = Some(resource.observation.observed_at_ms); + resource.observation.health = None; + } + (resource.observation.clone(), changed) + }; + if changed { + self.record_effect(FaultOperation::Stop); + } + self.boundary(FaultOperation::Stop, FaultBoundary::AfterEffect) + .await?; + Ok(observation) + } + + async fn remove( + &self, + unit: &RuntimeUnitRecord, + request: &RuntimeActionRequest, + ) -> RuntimeResult { + self.boundary(FaultOperation::Remove, FaultBoundary::BeforeEffect) + .await?; + let existed = { + let mut resources = self.resources.lock().expect("provider resource lock"); + let removed = resources + .get_mut(&unit.spec.unit_id) + .and_then(|generations| generations.remove(&unit.spec.generation)) + .is_some(); + if resources + .get(&unit.spec.unit_id) + .is_some_and(BTreeMap::is_empty) + { + resources.remove(&unit.spec.unit_id); + } + removed + }; + if existed { + self.record_effect(FaultOperation::Remove); + } + let removal = RuntimeRemoval { + schema: RuntimeRemoval::SCHEMA.into(), + request_id: request.request_id.clone(), + unit_id: unit.spec.unit_id.clone(), + generation: unit.spec.generation, + removed_at_ms: unit.observation.observed_at_ms.saturating_add(1), + already_absent: !existed, + }; + self.boundary(FaultOperation::Remove, FaultBoundary::AfterEffect) + .await?; + Ok(removal) + } + + async fn logs( + &self, + unit: &RuntimeUnitRecord, + _query: &RuntimeLogQuery, + ) -> RuntimeResult> { + self.boundary(FaultOperation::Logs, FaultBoundary::BeforeEffect) + .await?; + let chunks = vec![RuntimeLogChunk { + schema: RuntimeLogChunk::SCHEMA.into(), + cursor: "fault-cursor-1".into(), + sequence: 1, + observed_at_ms: unit.observation.observed_at_ms, + stream: RuntimeLogStream::Stdout, + data: "ready\n".into(), + }]; + self.boundary(FaultOperation::Logs, FaultBoundary::AfterEffect) + .await?; + Ok(chunks) + } + + async fn exec( + &self, + unit: &RuntimeUnitRecord, + request: &RuntimeExecRequest, + ) -> RuntimeResult { + self.boundary(FaultOperation::Exec, FaultBoundary::BeforeEffect) + .await?; + let key = Self::exec_key(request); + let (result, created) = { + let mut results = self.exec_results.lock().expect("exec result lock"); + if let Some(result) = results.get(&key) { + (result.clone(), false) + } else { + let result = RuntimeExecResult { + schema: RuntimeExecResult::SCHEMA.into(), + request_id: request.request_id.clone(), + observation: unit.observation.clone(), + exit_code: 0, + stdout: "ok\n".into(), + stderr: String::new(), + truncated: false, + }; + results.insert(key, result.clone()); + (result, true) + } + }; + if created { + self.record_effect(FaultOperation::Exec); + } + self.boundary(FaultOperation::Exec, FaultBoundary::AfterEffect) + .await?; + Ok(result) + } +} diff --git a/tests/support/fixtures.rs b/tests/support/fixtures.rs new file mode 100644 index 0000000..f54cb5e --- /dev/null +++ b/tests/support/fixtures.rs @@ -0,0 +1,76 @@ +use super::fault_driver::IMAGE_MEDIA_TYPE; +use a3s_runtime::contract::{ + ArtifactRef, IsolationLevel, NetworkMode, ResourceLimits, RestartPolicy, RuntimeActionRequest, + RuntimeApplyRequest, RuntimeExecRequest, RuntimeNetworkSpec, RuntimeProcessSpec, + RuntimeUnitClass, RuntimeUnitSpec, +}; +use std::collections::BTreeMap; + +pub fn service_spec(unit_id: &str) -> RuntimeUnitSpec { + RuntimeUnitSpec { + schema: RuntimeUnitSpec::SCHEMA.into(), + unit_id: unit_id.into(), + generation: 1, + class: RuntimeUnitClass::Service, + artifact: ArtifactRef { + uri: format!("oci://registry.example/a3s/fault@sha256:{}", "a".repeat(64)), + digest: format!("sha256:{}", "a".repeat(64)), + media_type: IMAGE_MEDIA_TYPE.into(), + }, + process: RuntimeProcessSpec { + command: vec!["/bin/service".into()], + args: Vec::new(), + working_directory: None, + environment: BTreeMap::new(), + }, + mounts: Vec::new(), + secrets: Vec::new(), + network: RuntimeNetworkSpec { + mode: NetworkMode::None, + ports: Vec::new(), + }, + resources: ResourceLimits { + cpu_millis: 100, + memory_bytes: 64 * 1024 * 1024, + pids: 32, + ephemeral_storage_bytes: None, + execution_timeout_ms: None, + }, + isolation: IsolationLevel::Container, + health: None, + restart: RestartPolicy::Always, + outputs: Vec::new(), + semantics_profile_digest: None, + } +} + +pub fn apply_request(request_id: &str, unit_id: &str) -> RuntimeApplyRequest { + RuntimeApplyRequest { + schema: RuntimeApplyRequest::SCHEMA.into(), + request_id: request_id.into(), + deadline_at_ms: None, + spec: service_spec(unit_id), + } +} + +pub fn action_request(request_id: &str, unit_id: &str) -> RuntimeActionRequest { + RuntimeActionRequest { + schema: RuntimeActionRequest::SCHEMA.into(), + request_id: request_id.into(), + unit_id: unit_id.into(), + generation: 1, + deadline_at_ms: None, + } +} + +pub fn exec_request(request_id: &str, unit_id: &str) -> RuntimeExecRequest { + RuntimeExecRequest { + schema: RuntimeExecRequest::SCHEMA.into(), + request_id: request_id.into(), + unit_id: unit_id.into(), + generation: 1, + command: vec!["/bin/true".into()], + timeout_ms: 1_000, + deadline_at_ms: None, + } +} diff --git a/tests/support/mod.rs b/tests/support/mod.rs new file mode 100644 index 0000000..333d99a --- /dev/null +++ b/tests/support/mod.rs @@ -0,0 +1,2 @@ +pub mod fault_driver; +pub mod fixtures;