diff --git a/.changeset/me-permissions-retry-transient.md b/.changeset/me-permissions-retry-transient.md new file mode 100644 index 0000000000..9c66e0b4bb --- /dev/null +++ b/.changeset/me-permissions-retry-transient.md @@ -0,0 +1,47 @@ +--- +"@object-ui/permissions": minor +"@object-ui/console": patch +--- + +fix(permissions,console): `MePermissionsProvider` retries a transient `/me/permissions` failure instead of stranding the app on its loading state + +"Not now" is a real answer from this endpoint. On a multi-tenant host it is served +by the environment kernel that owns the session, and a COLD one answers `503` + +`Retry-After` while it warms (objectstack#4159 / cloud#927). The provider treated +that like any other failure: it set `error` — and a consumer that passes no +`errorFallback` renders `loadingFallback` for the error state too. The console +does exactly that (`loadingFallback={}`, no `errorFallback`), so +the app sat on its spinner indefinitely, with a `retry` nobody could reach. + +The fetch now re-attempts a **transient** failure — `408`, `425`, `429`, `502`, +`503`, `504`, or a thrown fetch (offline / DNS / aborted), which never got an +answer at all. A server-stated `Retry-After` wins over the exponential backoff +(both wire forms are read, and clamped to 30s so a hostile value cannot park the +UI); otherwise the delay doubles from `retryBaseDelayMs`. `loading` stays true +across the waits, so the fail-closed loading state holds and consumers never see +a permissive flash mid-recovery. + +Unchanged for a real answer about the caller: `401`, `403`, `404` and `500` fail +on the first attempt exactly as before. `500` is deliberately not retried — a +genuine server fault neither benefits from hammering nor should be hidden behind +a spinner. + +**New props**, both optional and defaulted so no call site needs to change: + +- `maxRetries` (default `3`) — `0` restores the previous single-attempt + behaviour. +- `retryBaseDelayMs` (default `500`) — base for the exponential backoff. + +Also fixes a latent race the retries made much wider: the in-flight fetch is now +cancelled when the effect tears down, so a slow answer for a previous `endpoint` +or `fetcher` can no longer overwrite a fast answer for the current one. The retry +primitives (`parseRetryAfterMs`, `backoffMs`, `isTransientFailure`, +`TRANSIENT_STATUS`, `PermissionsFetchError`) live in a new internal `./retry` +module — not exported from the package root. + +**The console now passes an `errorFallback`.** Retrying narrows the window but +cannot close it — a kernel build slower than the retry budget still lands in the +error state, and rendering `loadingFallback` there is what produced the eternal +spinner. It now renders ``, using the +error + retry affordance that component has carried all along, so a user is never +left with a spinner and no way forward. diff --git a/apps/console/src/AppContent.tsx b/apps/console/src/AppContent.tsx index d10739e3e8..9c0732d455 100644 --- a/apps/console/src/AppContent.tsx +++ b/apps/console/src/AppContent.tsx @@ -132,7 +132,18 @@ export function AppContent() { // rendering fell open (restricted fields rendered editable). const authFetch = useMemo(() => createAuthenticatedFetch(), []); return ( - }> + } + // Without this, the provider's error state renders `loadingFallback` too — + // an eternal spinner with a `retry` nobody can reach. The provider already + // re-attempts a transient failure (a cold environment kernel answers 503 + + // `Retry-After` while it warms, objectstack#4159); this is what the user + // sees once those are exhausted, and `LoadingScreen` has carried the + // error + retry affordance all along. + errorFallback={(err, retry) => } + > diff --git a/packages/permissions/src/MePermissionsProvider.tsx b/packages/permissions/src/MePermissionsProvider.tsx index 9c096c5d6e..e4ee2a74cb 100644 --- a/packages/permissions/src/MePermissionsProvider.tsx +++ b/packages/permissions/src/MePermissionsProvider.tsx @@ -7,6 +7,13 @@ */ import React, { useEffect, useMemo, useState, useCallback } from 'react'; +import { + PermissionsFetchError, + backoffMs, + isTransientFailure, + parseRetryAfterMs, + sleep, +} from './retry'; import type { PermissionAction, PermissionCheckResult, @@ -65,6 +72,27 @@ export interface MePermissionsProviderProps { loadingFallback?: React.ReactNode; /** Rendered when load fails */ errorFallback?: (err: Error, retry: () => void) => React.ReactNode; + /** + * How many times to re-attempt a TRANSIENT failure before giving up — a + * response that says "not now" (`TRANSIENT_STATUS` in `./retry`) or a network + * error. `0` disables retrying. + * + * This exists because "not now" is a real answer from this endpoint. On a + * multi-tenant host it is served by the environment kernel that owns the + * session, and a cold one answers `503` + `Retry-After` while it warms + * (objectstack#4159). Without a retry the provider keeps `loadingFallback` + * on screen forever, because a consumer that passes no `errorFallback` — the + * console does exactly that — renders the loading node for the error state + * too, and nothing ever calls `retry`. + * + * @default 3 + */ + maxRetries?: number; + /** + * Base for the exponential backoff between retries, in ms. A `Retry-After` + * header on the response wins over it. @default 500 + */ + retryBaseDelayMs?: number; /** Children */ children: React.ReactNode; } @@ -88,34 +116,75 @@ export function MePermissionsProvider({ initialPermissions, loadingFallback = null, errorFallback, + maxRetries = 3, + retryBaseDelayMs = 500, children, }: MePermissionsProviderProps) { const [data, setData] = useState(initialPermissions ?? null); const [error, setError] = useState(null); const [loading, setLoading] = useState(!initialPermissions); - const fetchPermissions = useCallback(async () => { - setLoading(true); - setError(null); - try { + /** + * Run the fetch, re-attempting a transient failure. + * + * `token` cancels an in-flight attempt. Retries put real time (a backoff, or + * a server-stated `Retry-After`) between the request and the `setState`, and + * during that window the effect may be torn down — by an unmount, or by + * `endpoint`/`fetcher` changing. Without the token a superseded attempt would + * still land, so a slow answer for the OLD endpoint could overwrite a fast + * answer for the new one. + */ + const fetchPermissions = useCallback( + async (token?: { cancelled: boolean }) => { + const live = () => !token?.cancelled; + setLoading(true); + setError(null); const doFetch = fetcher ?? fetch; - const res = await doFetch(endpoint, { - credentials: 'include', - headers: { Accept: 'application/json' }, - }); - if (!res.ok) throw new Error(`Permissions endpoint returned ${res.status}`); - const json = (await res.json()) as MePermissionsResponse; - setData(json); - } catch (e) { - setError(e instanceof Error ? e : new Error(String(e))); - } finally { - setLoading(false); - } - }, [endpoint, fetcher]); + const attempts = Math.max(0, maxRetries) + 1; + for (let attempt = 0; attempt < attempts; attempt++) { + try { + const res = await doFetch(endpoint, { + credentials: 'include', + headers: { Accept: 'application/json' }, + }); + if (!res.ok) { + throw new PermissionsFetchError( + res.status, + parseRetryAfterMs(res.headers?.get?.('Retry-After'), Date.now()), + ); + } + const json = (await res.json()) as MePermissionsResponse; + if (!live()) return; + setData(json); + setLoading(false); + return; + } catch (e) { + const err = e instanceof Error ? e : new Error(String(e)); + if (!isTransientFailure(err) || attempt === attempts - 1) { + if (!live()) return; + setError(err); + setLoading(false); + return; + } + // `loading` stays true across the wait, so the fail-closed loading + // state holds and the recovery is invisible to consumers. + const stated = err instanceof PermissionsFetchError ? err.retryAfterMs : undefined; + await sleep(backoffMs(attempt, retryBaseDelayMs, stated)); + if (!live()) return; + } + } + }, + [endpoint, fetcher, maxRetries, retryBaseDelayMs], + ); + + /** The `retry` handed to `errorFallback` — uncancelled, it is user-initiated. */ + const retry = useCallback(() => { void fetchPermissions(); }, [fetchPermissions]); useEffect(() => { if (initialPermissions) return; - void fetchPermissions(); + const token = { cancelled: false }; + void fetchPermissions(token); + return () => { token.cancelled = true; }; }, [fetchPermissions, initialPermissions]); const checkField = useCallback( @@ -232,7 +301,7 @@ export function MePermissionsProvider({ if (loading && !data) return <>{loadingFallback}; if (error && !data) { - if (errorFallback) return <>{errorFallback(error, fetchPermissions)}; + if (errorFallback) return <>{errorFallback(error, retry)}; return <>{loadingFallback}; } diff --git a/packages/permissions/src/__tests__/MePermissionsProvider.retry.test.tsx b/packages/permissions/src/__tests__/MePermissionsProvider.retry.test.tsx new file mode 100644 index 0000000000..e4c9938718 --- /dev/null +++ b/packages/permissions/src/__tests__/MePermissionsProvider.retry.test.tsx @@ -0,0 +1,177 @@ +/** + * ObjectUI + * Copyright (c) 2024-present ObjectStack Inc. + * + * "Not now" is a real answer from `/me/permissions`, and it used to strand the + * console. + * + * On a multi-tenant host the endpoint is served by the environment kernel that + * owns the session, and a COLD one answers `503` + `Retry-After` while it warms + * (objectstack#4159 / cloud#927). The provider treated that like any other + * failure: it set `error`, and a consumer that passes no `errorFallback` — the + * console passes only `loadingFallback={}` — renders the + * loading node for the error state too. So the app sat on its spinner forever, + * with a `retry` nobody could reach. + * + * These pin the retry: transient answers are re-attempted (server-stated delay + * first), real answers about the caller are not, and the fail-closed loading + * state holds throughout rather than flashing permissive. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import React from 'react'; +import { MePermissionsProvider } from '../MePermissionsProvider'; +import { parseRetryAfterMs } from '../retry'; +import { usePermissions } from '../usePermissions'; + +function Probe() { + const { isLoaded, check } = usePermissions(); + return ( +
+ {String(isLoaded)} + {String(check('account', 'update').allowed)} +
+ ); +} + +const PAYLOAD = { + authenticated: true, + userId: 'u1', + tenantId: 't1', + roles: [], + permissionSets: ['member_default'], + objects: { account: { allowRead: true, allowEdit: false } }, + fields: {}, +}; + +const ok = () => ({ ok: true, status: 200, headers: new Headers(), json: async () => PAYLOAD }); +const fail = (status: number, headers: Record = {}) => ({ + ok: false, + status, + headers: new Headers(headers), + json: async () => ({}), +}); + +/** Render with retries fast enough that the tests need no timer control. */ +const renderProvider = (props: Record = {}) => + render( + + + , + ); + +describe('MePermissionsProvider retries a transient failure', () => { + beforeEach(() => { vi.spyOn(global, 'fetch' as any); }); + afterEach(() => { vi.restoreAllMocks(); }); + + it('recovers from a warming 503 and serves the real answer', async () => { + (global.fetch as any) + .mockResolvedValueOnce(fail(503, { 'Retry-After': '0' })) + .mockResolvedValueOnce(ok()); + + renderProvider(); + + await waitFor(() => expect(screen.getByTestId('loaded').textContent).toBe('true')); + // The permission answer really arrived — not the anonymous fail-open. + expect(screen.getByTestId('edit').textContent).toBe('false'); + expect((global.fetch as any)).toHaveBeenCalledTimes(2); + }); + + it('prefers the server-stated Retry-After over its own backoff', async () => { + // Timer-free proof: the backoff base is 100s, so if the header were ignored + // this test could not finish. `Retry-After: 0` says "now". + (global.fetch as any) + .mockResolvedValueOnce(fail(503, { 'Retry-After': '0' })) + .mockResolvedValueOnce(ok()); + + renderProvider({ retryBaseDelayMs: 100_000 }); + + await waitFor(() => expect(screen.getByTestId('loaded').textContent).toBe('true')); + }); + + it('retries a thrown fetch (offline / DNS / aborted) too', async () => { + (global.fetch as any) + .mockRejectedValueOnce(new TypeError('Failed to fetch')) + .mockResolvedValueOnce(ok()); + + renderProvider(); + + await waitFor(() => expect(screen.getByTestId('loaded').textContent).toBe('true')); + }); + + it('holds the fail-closed loading state while retrying — never a permissive flash', async () => { + (global.fetch as any) + .mockResolvedValueOnce(fail(503)) + .mockResolvedValueOnce(fail(503)) + .mockResolvedValueOnce(ok()); + + renderProvider({ loadingFallback: }); + + // Children (and therefore any permissive default) are not rendered yet. + expect(screen.getByTestId('spinner')).toBeTruthy(); + expect(screen.queryByTestId('loaded')).toBeNull(); + await waitFor(() => expect(screen.getByTestId('loaded').textContent).toBe('true')); + }); + + it('gives up after maxRetries and surfaces the error', async () => { + (global.fetch as any).mockResolvedValue(fail(503)); + + const errorFallback = vi.fn((err: Error) => {err.message}); + renderProvider({ maxRetries: 2, errorFallback }); + + await waitFor(() => expect(screen.getByTestId('err')).toBeTruthy()); + expect(screen.getByTestId('err').textContent).toContain('503'); + expect((global.fetch as any)).toHaveBeenCalledTimes(3); // 1 attempt + 2 retries + }); +}); + +describe('MePermissionsProvider does NOT retry a real answer about the caller', () => { + beforeEach(() => { vi.spyOn(global, 'fetch' as any); }); + afterEach(() => { vi.restoreAllMocks(); }); + + it.each([401, 403, 404, 500])('gives up immediately on %i', async (status) => { + (global.fetch as any).mockResolvedValue(fail(status)); + + const errorFallback = vi.fn((err: Error) => {err.message}); + renderProvider({ errorFallback }); + + await waitFor(() => expect(screen.getByTestId('err')).toBeTruthy()); + expect((global.fetch as any)).toHaveBeenCalledTimes(1); + }); + + it('maxRetries: 0 disables retrying entirely', async () => { + (global.fetch as any).mockResolvedValue(fail(503)); + + const errorFallback = vi.fn((err: Error) => {err.message}); + renderProvider({ maxRetries: 0, errorFallback }); + + await waitFor(() => expect(screen.getByTestId('err')).toBeTruthy()); + expect((global.fetch as any)).toHaveBeenCalledTimes(1); + }); +}); + +describe('parseRetryAfterMs', () => { + const NOW = Date.parse('2026-07-30T12:00:00Z'); + + it('reads delta-seconds', () => { + expect(parseRetryAfterMs('4', NOW)).toBe(4000); + expect(parseRetryAfterMs('0', NOW)).toBe(0); + expect(parseRetryAfterMs(' 7 ', NOW)).toBe(7000); + }); + + it('reads an HTTP-date, relative to now', () => { + expect(parseRetryAfterMs('Thu, 30 Jul 2026 12:00:03 GMT', NOW)).toBe(3000); + }); + + it('clamps a far-future value so it cannot park the UI', () => { + expect(parseRetryAfterMs('999999', NOW)).toBe(30_000); + expect(parseRetryAfterMs('Fri, 31 Jul 2026 12:00:00 GMT', NOW)).toBe(30_000); + }); + + it('ignores absent, empty, unparseable and past values', () => { + for (const v of [null, undefined, '', ' ', 'soon', 'Thu, 30 Jul 2026 11:59:00 GMT']) { + expect(parseRetryAfterMs(v, NOW), String(v)).toBeUndefined(); + } + }); +}); diff --git a/packages/permissions/src/retry.ts b/packages/permissions/src/retry.ts new file mode 100644 index 0000000000..b59557689f --- /dev/null +++ b/packages/permissions/src/retry.ts @@ -0,0 +1,83 @@ +/** + * ObjectUI + * Copyright (c) 2024-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * Retry primitives for {@link MePermissionsProvider}'s `/me/permissions` fetch. + * + * Their own module rather than the component's: they are plain functions with + * plain tests, and a component file that also exports helpers breaks fast + * refresh. + */ + +/** Upper bound on any single backoff wait, so a bad `Retry-After` cannot park the UI. */ +export const MAX_RETRY_DELAY_MS = 30_000; + +/** + * Statuses that mean "not now", so retrying can succeed without anything + * changing on the caller's side: + * + * - `408` / `425` — the request itself did not land. + * - `429` — rate limited; `Retry-After` usually accompanies it. + * - `502` / `503` / `504` — the upstream is absent, warming or timing out. + * `503` is the load-bearing one here: on a multi-tenant host this endpoint is + * served by the environment kernel that owns the session, and the framework + * answers `503` + `Retry-After` while a cold one is still being built + * (objectstack#4159). + * + * Deliberately NOT retried: `401` / `403` (a real answer about this caller), + * `404` (no such endpoint — retrying cannot conjure one) and `500` (a genuine + * server fault; hammering it neither helps nor is honest about the failure). + */ +export const TRANSIENT_STATUS: ReadonlySet = new Set([408, 425, 429, 502, 503, 504]); + +/** An `Error` that also carries the HTTP status it came from. */ +export class PermissionsFetchError extends Error { + constructor(readonly status: number, readonly retryAfterMs?: number) { + super(`Permissions endpoint returned ${status}`); + this.name = 'PermissionsFetchError'; + } +} + +/** + * Whether a failed attempt is worth re-attempting. A THROWN fetch (offline, + * DNS, aborted connection) is transient the same way a `503` is — the request + * never got an answer at all. + */ +export function isTransientFailure(err: unknown): boolean { + return err instanceof PermissionsFetchError ? TRANSIENT_STATUS.has(err.status) : true; +} + +/** + * `Retry-After` in ms, or `undefined` when absent/unparseable. Accepts both wire + * forms (delta-seconds and an HTTP-date) and clamps to {@link MAX_RETRY_DELAY_MS} + * so a hostile or buggy value cannot park the UI on its loading state for hours. + */ +export function parseRetryAfterMs(value: string | null | undefined, nowMs: number): number | undefined { + if (!value) return undefined; + const trimmed = value.trim(); + if (!trimmed) return undefined; + let ms: number; + if (/^\d+$/.test(trimmed)) { + ms = Number(trimmed) * 1000; + } else { + const at = Date.parse(trimmed); + if (Number.isNaN(at)) return undefined; + ms = at - nowMs; + } + if (!Number.isFinite(ms) || ms < 0) return undefined; + return Math.min(ms, MAX_RETRY_DELAY_MS); +} + +/** How long to wait before attempt `attempt + 1`. A server-stated delay wins. */ +export function backoffMs(attempt: number, baseDelayMs: number, statedMs?: number): number { + if (statedMs !== undefined) return statedMs; + return Math.min(baseDelayMs * 2 ** attempt, MAX_RETRY_DELAY_MS); +} + +export const sleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms));