Skip to content

Commit 95145b8

Browse files
AkarithosAkarithos
committed
Negotiate Content-Encoding for replay blocks and decode zstd client-side when stripped
Clients that do not advertise zstd in Accept-Encoding (notably plain-HTTP Chrome, which restricts zstd to HTTPS) were getting ERR_CONTENT_DECODING_FAILED on replay block loads because both identd and the documented Caddy file_server bypass set Content-Encoding: zstd unconditionally. identd now reads Accept-Encoding before serving a block. If the client accepts zstd, behavior is unchanged. If it does not, identd still passes through the raw zstd bytes but omits the Content- Encoding header so the browser delivers the payload unmodified. Server never decompresses — a JS-side decoder handles the fallback to avoid creating a CPU amplification vector. The frontend now reads block responses as bytes, detects the zstd frame magic on the first four, and routes through fzstd when needed (otherwise parses straight through as JSON). A 32 MB cap on the decompressed output catches runaway producer bugs. fzstd is decompress-only (16 KB raw / 7 KB gzipped bundle add) and depends on nothing else. Updates the README Caddyfile example with an Accept-Encoding matcher so its file_server bypass behaves the same way.
1 parent 0c14dce commit 95145b8

10 files changed

Lines changed: 517 additions & 10 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -367,10 +367,12 @@ directory behind the reverse proxy and let the proxy serve finalized
367367
`.json.zst` files directly:
368368

369369
```caddyfile
370+
@accepts_zstd header Accept-Encoding *zstd*
371+
370372
handle_path /api/replay/blocks/* {
371373
root * /var/lib/ident/replay/blocks
372-
header Content-Type application/json
373-
header Content-Encoding zstd
374+
header Content-Type application/octet-stream
375+
header @accepts_zstd Content-Encoding zstd
374376
header Cache-Control "public, max-age=31536000, immutable"
375377
file_server
376378
}

‎ident/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
"dependencies": {
2929
"cmdk": "^1.1.1",
3030
"country-flag-icons": "^1.6.17",
31+
"fzstd": "^0.1.1",
3132
"lucide-react": "^1.16.0",
3233
"react": "^19.2.6",
3334
"react-dom": "^19.2.6",

‎ident/pnpm-lock.yaml‎

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎ident/scripts/bench-fzstd.mjs‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
// Bench: realistic replay block decompress + magic-byte check overhead.
2+
// Block model: 5 min @ 1Hz = 300 frames; ~80 aircraft per frame; per-aircraft
3+
// payload ~10 fields. Compresses well; representative of identd output.
4+
import { decompress as fzstdDecompress } from "fzstd";
5+
import { performance } from "node:perf_hooks";
6+
import * as zlib from "node:zlib";
7+
8+
const RNG = (() => {
9+
let s = 0xdeadbeef;
10+
return () => ((s = (s * 16807) % 2147483647) / 2147483647);
11+
})();
12+
13+
// Production blocks observed at ~190 KB compressed. Identd records deltas,
14+
// not per-second snapshots, so the entry count is far lower than naively
15+
// "frameCount * aircraftPerFrame". Tune to match observed wire size.
16+
function makeBlock(frameCount = 75, aircraftPerFrame = 15) {
17+
const hexs = Array.from({ length: 256 }, (_, i) =>
18+
(0xa00000 + i).toString(16).padStart(6, "0"),
19+
);
20+
const frames = [];
21+
let now = 1779156900;
22+
for (let f = 0; f < frameCount; f++) {
23+
const aircraft = [];
24+
for (let i = 0; i < aircraftPerFrame; i++) {
25+
aircraft.push({
26+
hex: hexs[(f + i) % hexs.length],
27+
seen: 0.1 + RNG() * 0.5,
28+
lat: 37 + RNG() * 3,
29+
lon: -122 - RNG() * 3,
30+
altBaro: 1000 + Math.floor(RNG() * 39000),
31+
gs: Math.floor(RNG() * 500),
32+
track: Math.floor(RNG() * 360),
33+
squawk: String(1000 + Math.floor(RNG() * 7000)),
34+
flight: "AAL" + Math.floor(RNG() * 9999),
35+
category: "A3",
36+
});
37+
}
38+
frames.push({ now: now + f, aircraft });
39+
}
40+
return { version: 2, frames };
41+
}
42+
43+
async function main() {
44+
const block = makeBlock();
45+
const json = JSON.stringify(block);
46+
const jsonBytes = Buffer.from(json, "utf-8");
47+
48+
// zstd via node:zlib (Node 22+ has zstd). Falls back to gzip if unavailable.
49+
let compressed;
50+
if (zlib.zstdCompressSync) {
51+
compressed = zlib.zstdCompressSync(jsonBytes, {
52+
params: { [zlib.constants.ZSTD_c_compressionLevel]: 10 },
53+
});
54+
} else {
55+
console.error("node:zlib has no zstd; falling back to gzip baseline only");
56+
compressed = zlib.gzipSync(jsonBytes, { level: 9 });
57+
}
58+
59+
console.log(`JSON size: ${(jsonBytes.length / 1024).toFixed(1)} KB`);
60+
console.log(`zstd compressed: ${(compressed.length / 1024).toFixed(1)} KB (ratio ${(compressed.length / jsonBytes.length).toFixed(3)})`);
61+
console.log("");
62+
63+
const u8 = new Uint8Array(
64+
compressed.buffer,
65+
compressed.byteOffset,
66+
compressed.byteLength,
67+
);
68+
69+
// Warm up
70+
for (let i = 0; i < 5; i++) {
71+
const out = fzstdDecompress(u8);
72+
JSON.parse(new TextDecoder().decode(out));
73+
}
74+
75+
// Magic byte check
76+
function magicCheck(buf) {
77+
return (
78+
buf[0] === 0x28 &&
79+
buf[1] === 0xb5 &&
80+
buf[2] === 0x2f &&
81+
buf[3] === 0xfd
82+
);
83+
}
84+
const MAGIC_ITERS = 1_000_000;
85+
let _sink = 0;
86+
const t0 = performance.now();
87+
for (let i = 0; i < MAGIC_ITERS; i++) {
88+
if (magicCheck(u8)) _sink++;
89+
}
90+
const t1 = performance.now();
91+
console.log(
92+
`magic-byte check: ${((t1 - t0) / MAGIC_ITERS * 1e6).toFixed(2)} ns/op (${MAGIC_ITERS.toLocaleString()} iterations)`,
93+
);
94+
95+
// fzstd decompress only
96+
const DECOMP_ITERS = 100;
97+
const decompTimes = [];
98+
for (let i = 0; i < DECOMP_ITERS; i++) {
99+
const s = performance.now();
100+
fzstdDecompress(u8);
101+
decompTimes.push(performance.now() - s);
102+
}
103+
decompTimes.sort((a, b) => a - b);
104+
const p50 = decompTimes[Math.floor(decompTimes.length * 0.5)];
105+
const p95 = decompTimes[Math.floor(decompTimes.length * 0.95)];
106+
const p99 = decompTimes[Math.floor(decompTimes.length * 0.99)];
107+
const mean = decompTimes.reduce((a, b) => a + b, 0) / decompTimes.length;
108+
console.log(
109+
`fzstd decompress: mean ${mean.toFixed(2)} ms p50 ${p50.toFixed(2)} p95 ${p95.toFixed(2)} p99 ${p99.toFixed(2)} (${DECOMP_ITERS} runs)`,
110+
);
111+
112+
// decompress + TextDecoder + JSON.parse (full path)
113+
const FULL_ITERS = 100;
114+
const fullTimes = [];
115+
for (let i = 0; i < FULL_ITERS; i++) {
116+
const s = performance.now();
117+
const out = fzstdDecompress(u8);
118+
JSON.parse(new TextDecoder().decode(out));
119+
fullTimes.push(performance.now() - s);
120+
}
121+
fullTimes.sort((a, b) => a - b);
122+
const fp50 = fullTimes[Math.floor(fullTimes.length * 0.5)];
123+
const fp95 = fullTimes[Math.floor(fullTimes.length * 0.95)];
124+
const fp99 = fullTimes[Math.floor(fullTimes.length * 0.99)];
125+
const fmean = fullTimes.reduce((a, b) => a + b, 0) / fullTimes.length;
126+
console.log(
127+
`decompress+parse: mean ${fmean.toFixed(2)} ms p50 ${fp50.toFixed(2)} p95 ${fp95.toFixed(2)} p99 ${fp99.toFixed(2)} (${FULL_ITERS} runs)`,
128+
);
129+
130+
// Baseline: TextDecoder + JSON.parse on already-decompressed JSON (the "browser already decoded" path)
131+
const BASE_ITERS = 100;
132+
const baseTimes = [];
133+
for (let i = 0; i < BASE_ITERS; i++) {
134+
const s = performance.now();
135+
JSON.parse(new TextDecoder().decode(jsonBytes));
136+
baseTimes.push(performance.now() - s);
137+
}
138+
baseTimes.sort((a, b) => a - b);
139+
const bp50 = baseTimes[Math.floor(baseTimes.length * 0.5)];
140+
const bp95 = baseTimes[Math.floor(baseTimes.length * 0.95)];
141+
console.log(
142+
`parse-only (HTTPS path): mean ${(baseTimes.reduce((a, b) => a + b, 0) / baseTimes.length).toFixed(2)} ms p50 ${bp50.toFixed(2)} p95 ${bp95.toFixed(2)} (${BASE_ITERS} runs)`,
143+
);
144+
145+
// Scrub simulation: load N blocks back-to-back
146+
const SCRUB_BLOCKS = 30;
147+
const scrubTimes = [];
148+
for (let trial = 0; trial < 10; trial++) {
149+
const s = performance.now();
150+
for (let i = 0; i < SCRUB_BLOCKS; i++) {
151+
const out = fzstdDecompress(u8);
152+
JSON.parse(new TextDecoder().decode(out));
153+
}
154+
scrubTimes.push(performance.now() - s);
155+
}
156+
scrubTimes.sort((a, b) => a - b);
157+
console.log("");
158+
console.log(`Scrub simulation: ${SCRUB_BLOCKS} blocks back-to-back`);
159+
console.log(
160+
` total time: mean ${(scrubTimes.reduce((a, b) => a + b, 0) / scrubTimes.length).toFixed(1)} ms p50 ${scrubTimes[5].toFixed(1)} p95 ${scrubTimes[9].toFixed(1)}`,
161+
);
162+
console.log(` per block: ${((scrubTimes[5] / SCRUB_BLOCKS)).toFixed(2)} ms`);
163+
164+
if (_sink === 0) console.log(""); // keep _sink alive
165+
}
166+
167+
main().catch(console.error);

‎ident/src/data/replay.test.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,17 @@ function replayBlock() {
687687
}
688688

689689
function responseJson(body: unknown): Response {
690-
return { ok: true, status: 200, json: async () => body } as Response;
690+
return {
691+
ok: true,
692+
status: 200,
693+
json: async () => body,
694+
// Replay block loads read the response as bytes and decide whether to
695+
// zstd-decompress based on a magic-byte prefix check. The mock surfaces
696+
// the JSON serialization here so the no-zstd-magic path parses straight
697+
// through; tests that exercise the zstd path can override arrayBuffer.
698+
arrayBuffer: async () =>
699+
new TextEncoder().encode(JSON.stringify(body)).buffer,
700+
} as Response;
691701
}
692702

693703
function abortError(): DOMException {

‎ident/src/data/replay.ts‎

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
import { appPath } from "./basePath";
2+
import {
3+
decodeReplayBlockResponse,
4+
ReplayBlockBodyError,
5+
} from "./replayBlockBody";
26
import { replayFollowsLiveEdge, useIdentStore } from "./store";
37
import type {
48
ReplayBlockFile,
@@ -188,14 +192,13 @@ function loadReplayBlock(block: ReplayBlockIndex): Promise<void> | null {
188192
try {
189193
if (useIdentStore.getState().replay.cache[block.url]) return;
190194
const url = appPath(block.url.replace(/^\//, ""));
191-
const body = await fetchJson<ReplayBlockFile>(
195+
const bytes = await fetchReplayBlockBytes(
192196
url,
193-
{
194-
cache: "force-cache",
195-
},
197+
{ cache: "force-cache" },
196198
controller.signal,
197199
);
198200
if (controller.signal.aborted) throw new ReplayLoadCanceled();
201+
const body = decodeReplayBlockResponse(bytes) as ReplayBlockFile;
199202
if (body.version !== 2 || !Array.isArray(body.frames)) {
200203
throw new ReplayBlockFormatError(block.url);
201204
}
@@ -207,6 +210,9 @@ function loadReplayBlock(block: ReplayBlockIndex): Promise<void> | null {
207210
) {
208211
throw err;
209212
}
213+
if (err instanceof ReplayBlockBodyError) {
214+
throw new ReplayBlockFormatError(block.url);
215+
}
210216
throw new ReplayBlockLoadError(block.url, err);
211217
} finally {
212218
if (blockLoads.get(block.url)?.promise === load) {
@@ -280,6 +286,46 @@ function manifestIndexByUrl(blocks: ReplayBlockIndex[]): Map<string, number> {
280286
return next;
281287
}
282288

289+
// fetchReplayBlockBytes returns the raw response body for a block URL. The
290+
// caller decides whether to parse as JSON or zstd-decompress-then-parse via
291+
// decodeReplayBlockResponse — we cannot tell from the response which form
292+
// the server delivered, since modern browsers strip Content-Encoding once
293+
// they've natively decoded.
294+
async function fetchReplayBlockBytes(
295+
url: string,
296+
init: RequestInit,
297+
abortSignal: AbortSignal,
298+
): Promise<Uint8Array> {
299+
const controller = new AbortController();
300+
let timedOut = false;
301+
let canceled = false;
302+
const timeout = setTimeout(() => {
303+
timedOut = true;
304+
controller.abort();
305+
}, REPLAY_FETCH_TIMEOUT_MS);
306+
const abort = () => {
307+
canceled = true;
308+
controller.abort();
309+
};
310+
if (abortSignal.aborted) throw new ReplayLoadCanceled();
311+
abortSignal.addEventListener("abort", abort, { once: true });
312+
try {
313+
const res = await fetch(url, { ...init, signal: controller.signal });
314+
if (!res.ok) throw new Error(`Replay request failed: ${res.status}`);
315+
return new Uint8Array(await res.arrayBuffer());
316+
} catch (err) {
317+
if (err instanceof DOMException && err.name === "AbortError") {
318+
if (canceled || abortSignal.aborted) throw new ReplayLoadCanceled();
319+
if (!timedOut) throw new ReplayLoadCanceled();
320+
throw new Error("Replay request timed out");
321+
}
322+
throw err;
323+
} finally {
324+
clearTimeout(timeout);
325+
abortSignal.removeEventListener("abort", abort);
326+
}
327+
}
328+
283329
async function fetchJson<T>(
284330
url: string,
285331
init: RequestInit,

0 commit comments

Comments
 (0)