From 08deb4a46c87f520d76f96a07bdd7c25d4e0f077 Mon Sep 17 00:00:00 2001 From: GTC6244 <95836911+GTC6244@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:27:32 -0400 Subject: [PATCH] fix(api,k6): ride out chain read-after-write races that flake the staging deploy gate Three consecutive Deploy Staging runs (and the Jul 20 one) were killed by k6-correctness flakes that all reduce to reads racing just-mined writes on a load-balanced RPC: - POST /new_account 500 "Simulation failed: NoAccountAccess": the newAccount receipt proves the account exists, but send_transaction dry-runs the follow-up registerWalletDerivation via eth_call, which the provider can serve from a node that hasn't executed that block yet. That stale-read revert is the only way NoAccountAccess can occur at this call site, so retry it (4 attempts, 250ms exponential backoff); everything else still fails immediately. This is a real user-facing 500, not just a test flake. - integration.spec.ts sent update_action_metadata a hash of 0x0 when the list_actions read immediately after add_action/add_action_to_group was stale. Poll both list reads (bounded, 5 attempts / 2s) until the action is visible with a non-zero id, and fail explicitly if it never appears instead of forwarding 0x0. The third failure mode (every request EOF ~6 min after the cold box booted, run 29882923105) is not addressed here: the box passed wait-for-api, smoke, attestation and openapi at minute 2 and was unreachable by minute 7, and the ping-pong rollback correctly restored the domain to the old instance. Runs 1 and 2 stopped their cold box before minute 5, so they can't rule a dies-after-boot defect in or out; the next deploy is the second data point. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 4 ++ k6/correctness/integration.spec.ts | 59 +++++++++++++++++-- lit-api-server/src/core/account_management.rs | 50 +++++++++++++--- 3 files changed, 99 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9af27799..080fb2b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,10 @@ doesn't describe endpoints the released server lacks. - `CONTRIBUTING.md`, expanded `.env.example`, and per-component READMEs. ### Fixed +- `POST /new_account` no longer intermittently returns a 500 + (`NoAccountAccess`) when the RPC provider serves the follow-up wallet + registration's dry-run from a node that hasn't executed the just-mined + `newAccount` block; the propagation window is now retried with backoff. - Release builds no longer report a spurious `-modified` version suffix (`.dockerignore` excluded a tracked file, dirtying `git describe` inside the image build). diff --git a/k6/correctness/integration.spec.ts b/k6/correctness/integration.spec.ts index 91b0010f..c70296f1 100644 --- a/k6/correctness/integration.spec.ts +++ b/k6/correctness/integration.spec.ts @@ -6,6 +6,7 @@ * k6 run k6/integration.spec.ts * BASE_URL=https://your-instance.phala.network/core/v1 k6 run k6/integration.spec.ts */ +import { sleep } from "k6"; import { checkAndLog, warnOnHttpFailures } from "../helpers.ts"; import { LitApiServerClient } from "../litApiServer.ts"; import { PRECREATED_ACCOUNTS } from "../setup.ts"; @@ -206,10 +207,41 @@ export default function (data: IntegrationSetupData) { }, "addActionToGroup"); // ── 10a. listActions (account-level, no group_id) ──────────────────────── - const listActionsAccountRes = client.listActions( + // The action metadata + group membership were written on-chain moments ago; + // list reads can briefly lag the write (the RPC can serve the read from a + // node that hasn't executed those blocks yet). Poll until the action is + // visible with a real id so propagation lag doesn't fail the deploy gate — + // a genuinely missing action still fails the checks after the retries, and + // a stale group listing would otherwise send hash 0x0 to + // update_action_metadata below. + const LIST_ACTIONS_ATTEMPTS = 5; + const findHelloWorld = (body: unknown): { id: string; name: string } | undefined => { + try { + const items = JSON.parse(body as string) as { id: string; name: string }[]; + const item = Array.isArray(items) ? items.find((a) => a.name === "hello-world") : undefined; + // An all-zero id means the membership row landed but the metadata read + // is still stale — treat as not-yet-visible and keep polling. + return item && item.id && !/^(0x)?0*$/.test(item.id) ? item : undefined; + } catch { + return undefined; + } + }; + + let listActionsAccountRes = client.listActions( { page_number: 0, page_size: 10 }, authHeaders, ); + for ( + let i = 1; + i < LIST_ACTIONS_ATTEMPTS && !findHelloWorld(listActionsAccountRes.response.body); + i++ + ) { + sleep(2); + listActionsAccountRes = client.listActions( + { page_number: 0, page_size: 10 }, + authHeaders, + ); + } if (!assertOk("listActionsAccount", "GET /list_actions (account)", listActionsAccountRes)) return; checkAndLog(listActionsAccountRes.response, { "listActionsAccount returns array": (r) => { @@ -229,11 +261,22 @@ export default function (data: IntegrationSetupData) { }, }, "listActionsAccount"); - // ── 10b. listActions (in group) ───────────────────────────────────────── - const listActionsRes = client.listActions( + // ── 10b. listActions (in group) — same propagation-lag polling as 10a ──── + let listActionsRes = client.listActions( { group_id: parseInt(groupId), page_number: 0, page_size: 10 }, authHeaders, ); + for ( + let i = 1; + i < LIST_ACTIONS_ATTEMPTS && !findHelloWorld(listActionsRes.response.body); + i++ + ) { + sleep(2); + listActionsRes = client.listActions( + { group_id: parseInt(groupId), page_number: 0, page_size: 10 }, + authHeaders, + ); + } if (!assertOk("listActions", "GET /list_actions", listActionsRes)) return; checkAndLog(listActionsRes.response, { "listActions returns array": (r) => { @@ -244,8 +287,14 @@ export default function (data: IntegrationSetupData) { } }, }, "listActions"); - const listActionsBody = JSON.parse(listActionsRes.response.body as string) as { id: string; name: string }[]; - const actionItem = listActionsBody.find((a) => a.name === "hello-world"); + const actionItem = findHelloWorld(listActionsRes.response.body); + if ( + !checkAndLog(listActionsRes.response, { + "listActions group contains added action with non-zero id": () => actionItem !== undefined, + }, "listActions") + ) { + return; + } const hashedCid = actionItem?.id ?? ""; // ── 10. addPkpToGroup ───────────────────────────────────────────────────── const addPkpRes = client.addPkpToGroup( diff --git a/lit-api-server/src/core/account_management.rs b/lit-api-server/src/core/account_management.rs index 94015ed6..9c8ecf64 100644 --- a/lit-api-server/src/core/account_management.rs +++ b/lit-api-server/src/core/account_management.rs @@ -42,6 +42,11 @@ use rocket::serde::json::Json; /// to include the NotAllowedTo* error types. Track as follow-up. const PERMISSION_ERROR_PATTERNS: &[&str] = &["NotAllowedTo", "NotMasterAccount", "NoAccountAccess"]; +/// Attempts for the registerWalletDerivation follow-up write inside `new_account` +/// (1 initial + 3 retries, exponential backoff from 250ms). Only the stale-read +/// NoAccountAccess revert is retried — see the comment at the call site. +const REGISTER_WALLET_DERIVATION_ATTEMPTS: u32 = 4; + fn map_contract_error(e: anyhow::Error, context: &str) -> ApiStatus { let msg = format!("{}", e); if PERMISSION_ERROR_PATTERNS @@ -125,15 +130,42 @@ pub async fn new_account( } // technically this is NOT a derivaton path at all, but it's a stand-in for now - accounts::register_wallet_derivation( - signer_pool, - &api_key, - wallet_address, - derivation_path, - "AMW", - "Account Master Wallet", - ) - .await?; + // + // The newAccount receipt above proves the account exists on-chain, but + // send_transaction dry-runs this follow-up write via eth_call, and the RPC + // provider can serve that call from a node that hasn't executed the + // newAccount block yet — reverting NoAccountAccess for an account that was + // just mined. That stale-read window is the only way NoAccountAccess can + // occur here, so retry it briefly; any other error fails immediately. + let mut attempt = 0u32; + loop { + attempt += 1; + match accounts::register_wallet_derivation( + signer_pool.clone(), + &api_key, + wallet_address, + derivation_path, + "AMW", + "Account Master Wallet", + ) + .await + { + Ok(_) => break, + Err(e) + if attempt < REGISTER_WALLET_DERIVATION_ATTEMPTS + && e.to_string().contains("NoAccountAccess") => + { + let backoff_ms = 250u64 << (attempt - 1); + tracing::warn!( + "new_account: registerWalletDerivation simulation hit stale-read \ + NoAccountAccess (attempt {attempt}/{REGISTER_WALLET_DERIVATION_ATTEMPTS}); \ + retrying in {backoff_ms}ms" + ); + tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; + } + Err(e) => return Err(e.into()), + } + } // Best-effort: eagerly create the Stripe customer (with $0 balance), set the email // if provided, and grant starter credits when STARTER_CREDITS_CENTS is configured.