diff --git a/packages/core/src/files.test.ts b/packages/core/src/files.test.ts new file mode 100644 index 00000000..1a353c7c --- /dev/null +++ b/packages/core/src/files.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; + +import { hasOversizedPathSegment, MAX_PATH_SEGMENT_BYTES, writeFile } from "./files.js"; +import type { StorageAdapter } from "./storage.js"; + +describe("hasOversizedPathSegment", () => { + it("accepts a path whose every segment is at the byte limit", () => { + const segment = "a".repeat(MAX_PATH_SEGMENT_BYTES - ".json".length); + expect(hasOversizedPathSegment(`/reddit/subreddits/localllama/posts/${segment}.json`)).toBe(false); + }); + + it("rejects a path with one segment over the byte limit", () => { + const segment = "a".repeat(MAX_PATH_SEGMENT_BYTES + 1); + expect(hasOversizedPathSegment(`/reddit/subreddits/localllama/posts/${segment}.json`)).toBe(true); + }); + + it("counts UTF-8 encoded bytes, not JS string length", () => { + // Each "é" is 1 UTF-16 code unit but 2 UTF-8 bytes. + const segment = "é".repeat(MAX_PATH_SEGMENT_BYTES); // 200 chars, 400 bytes + expect(hasOversizedPathSegment(`/x/${segment}`)).toBe(true); + }); + + it("ignores empty segments from a normalized path", () => { + expect(hasOversizedPathSegment("/reddit/subreddits/localllama.json")).toBe(false); + }); + + it("reproduces the real oversized r/LocalLLaMA post slug that broke the mount", () => { + const badPath = + "/reddit/subreddits/localllama/posts/i-just-don-t-get-it-these-big-tech-companies-can-illegally-" + + "scrape-the-entire-internet-and-gatekeep-their-better-models-behind-higher-prices-so-it-s-natural-" + + "that-people-look-for-affordable-options-and-there-will-be-providers-who-apparently-distill-models-" + + "from-them__1uvwn9q.json"; + expect(hasOversizedPathSegment(badPath)).toBe(true); + }); +}); + +describe("writeFile", () => { + function storageStub(overrides: Partial = {}): StorageAdapter { + return { + getFile: () => null, + listFiles: () => [], + putFile: () => {}, + deleteFile: () => {}, + appendEvent: () => {}, + listEvents: () => ({ items: [], nextCursor: null }), + getRecentEvents: () => [], + getOperation: () => null, + putOperation: () => {}, + listOperations: () => ({ items: [], nextCursor: null }), + nextRevision: () => "rev_1", + nextOperationId: () => "op_1", + nextEventId: () => "evt_1", + enqueueWriteback: () => {}, + getPendingWritebacks: () => [], + getWorkspaceId: () => "ws_test", + ...overrides, + } as StorageAdapter; + } + + it("rejects a write whose path has an oversized segment before touching storage", () => { + const segment = "a".repeat(MAX_PATH_SEGMENT_BYTES + 1); + let storageTouched = false; + const storage = storageStub({ + getFile: () => { + storageTouched = true; + return null; + }, + }); + + const result = writeFile(storage, { + path: `/reddit/subreddits/localllama/posts/${segment}.json`, + ifMatch: "*", + content: "{}", + }); + + expect(result).toEqual({ ok: false, error: "invalid_input" }); + expect(storageTouched).toBe(false); + }); + + it("allows a write with normal-length path segments to proceed past the guard", () => { + const storage = storageStub(); + const result = writeFile(storage, { + path: "/reddit/subreddits/localllama/posts/a-normal-title__abc123.json", + ifMatch: "0", + content: "{}", + }); + + expect(result.ok).toBe(true); + }); +}); diff --git a/packages/core/src/files.ts b/packages/core/src/files.ts index ec0e4cbc..fdb06775 100644 --- a/packages/core/src/files.ts +++ b/packages/core/src/files.ts @@ -94,6 +94,9 @@ export function writeFile(storage: StorageAdapter, req: WriteFileRequest): Write } const path = normalizePath(req.path); + if (hasOversizedPathSegment(path)) { + return { ok: false, error: "invalid_input" }; + } const ifMatch = normalizeIfMatch(req.ifMatch); if (!ifMatch) { return { ok: false, error: "missing_precondition" }; @@ -274,6 +277,27 @@ export function deleteFile(storage: StorageAdapter, req: DeleteFileRequest): Del export const DEFAULT_CONTENT_TYPE = "text/markdown"; export const MAX_FILE_BYTES = 10 * 1024 * 1024; +// Local mounts materialize each path segment as a real filesystem entry, and +// most filesystems (ext4, APFS, ...) reject names over 255 bytes with +// ENAMETOOLONG -- worse once a mount's own atomic-write suffix (e.g. +// `.tmp-`) is appended to the leaf segment. A provider adapter that +// builds a path segment from unbounded user content (a long title, etc.) can +// produce a canonical path no mount can ever materialize, which then fails +// forever on every sync cycle since nothing else revisits an already-written +// path. Reject that at write time instead of a bounded content-derived slug +// having to be probabilistically clean everywhere it's used. +export const MAX_PATH_SEGMENT_BYTES = 200; + +function encodedByteLength(value: string): number { + return new TextEncoder().encode(value).length; +} + +export function hasOversizedPathSegment(path: string): boolean { + return path + .split("/") + .some((segment) => segment.length > 0 && encodedByteLength(segment) > MAX_PATH_SEGMENT_BYTES); +} + function normalizeEncoding(value?: string): "utf-8" | "base64" | null { const encoding = value?.trim().toLowerCase() ?? ""; if (!encoding || encoding === "utf-8" || encoding === "utf8") { diff --git a/packages/core/src/webhooks.test.ts b/packages/core/src/webhooks.test.ts index d461af97..9c8b88b1 100644 --- a/packages/core/src/webhooks.test.ts +++ b/packages/core/src/webhooks.test.ts @@ -208,6 +208,33 @@ describe("webhook Slack path canonicalization", () => { "/slack/channels/C123/messages/1711111111_000100/meta.json", ); }); + + it("rejects an envelope write whose path has an oversized segment instead of writing it", () => { + const storage = new MemoryStorage(); + const oversizedSlug = "a".repeat(250); + + const queued = ingestWebhook(storage, { + provider: "reddit", + eventType: "file.created", + path: `/reddit/subreddits/localllama/posts/${oversizedSlug}__1uvwn9q.json`, + deliveryId: "delivery_1", + timestamp: "2026-07-15T09:45:00.000Z", + correlationId: "corr_1", + }, { + generateEnvelopeId: () => "env_1", + coalesceWindowMs: 10_000, + }); + expect(queued.status).toBe("queued"); + + const envelope = storage.envelopes.get("env_1"); + if (!envelope) throw new Error("envelope not queued"); + + const result = applyWebhookEnvelope(storage, envelope); + + expect(result.status).toBe("rejected"); + expect(result.reason).toBe("path_too_long"); + expect(storage.files.size).toBe(0); + }); }); function fileRow(path: string, overrides: Partial = {}): FileRow { diff --git a/packages/core/src/webhooks.ts b/packages/core/src/webhooks.ts index 6d4fe7cb..ef384a3e 100644 --- a/packages/core/src/webhooks.ts +++ b/packages/core/src/webhooks.ts @@ -21,7 +21,7 @@ import type { Paginated, FileSemantics, } from "./storage.js"; -import { normalizePath, DEFAULT_CONTENT_TYPE, MAX_FILE_BYTES, encodedSize } from "./files.js"; +import { normalizePath, DEFAULT_CONTENT_TYPE, MAX_FILE_BYTES, encodedSize, hasOversizedPathSegment } from "./files.js"; export interface IngestWebhookInput { provider: string; @@ -397,6 +397,16 @@ export function applyWebhookEnvelope( }; } + if (hasOversizedPathSegment(event.path)) { + return { + status: "rejected", + eventType: event.type, + path: event.path, + revision: null, + reason: "path_too_long", + }; + } + const content = typeof event.content === "string" ? event.content