Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions packages/core/src/files.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import { describe, expect, it } from "vitest";

import { hasOversizedPathSegment, MAX_PATH_SEGMENT_BYTES, writeFile } from "./files.js";
import type { StorageAdapter } from "./storage.js";

describe("hasOversizedPathSegment", () => {
it("accepts a path whose every segment is at the byte limit", () => {
const segment = "a".repeat(MAX_PATH_SEGMENT_BYTES - ".json".length);
expect(hasOversizedPathSegment(`/reddit/subreddits/localllama/posts/${segment}.json`)).toBe(false);
});

it("rejects a path with one segment over the byte limit", () => {
const segment = "a".repeat(MAX_PATH_SEGMENT_BYTES + 1);
expect(hasOversizedPathSegment(`/reddit/subreddits/localllama/posts/${segment}.json`)).toBe(true);
});

it("counts UTF-8 encoded bytes, not JS string length", () => {
// Each "é" is 1 UTF-16 code unit but 2 UTF-8 bytes.
const segment = "é".repeat(MAX_PATH_SEGMENT_BYTES); // 200 chars, 400 bytes
expect(hasOversizedPathSegment(`/x/${segment}`)).toBe(true);
});

it("ignores empty segments from a normalized path", () => {
expect(hasOversizedPathSegment("/reddit/subreddits/localllama.json")).toBe(false);
});

it("reproduces the real oversized r/LocalLLaMA post slug that broke the mount", () => {
const badPath =
"/reddit/subreddits/localllama/posts/i-just-don-t-get-it-these-big-tech-companies-can-illegally-" +
"scrape-the-entire-internet-and-gatekeep-their-better-models-behind-higher-prices-so-it-s-natural-" +
"that-people-look-for-affordable-options-and-there-will-be-providers-who-apparently-distill-models-" +
"from-them__1uvwn9q.json";
expect(hasOversizedPathSegment(badPath)).toBe(true);
});
});

describe("writeFile", () => {
function storageStub(overrides: Partial<StorageAdapter> = {}): StorageAdapter {
return {
getFile: () => null,
listFiles: () => [],
putFile: () => {},
deleteFile: () => {},
appendEvent: () => {},
listEvents: () => ({ items: [], nextCursor: null }),
getRecentEvents: () => [],
getOperation: () => null,
putOperation: () => {},
listOperations: () => ({ items: [], nextCursor: null }),
nextRevision: () => "rev_1",
nextOperationId: () => "op_1",
nextEventId: () => "evt_1",
enqueueWriteback: () => {},
getPendingWritebacks: () => [],
getWorkspaceId: () => "ws_test",
...overrides,
} as StorageAdapter;
}

it("rejects a write whose path has an oversized segment before touching storage", () => {
const segment = "a".repeat(MAX_PATH_SEGMENT_BYTES + 1);
let storageTouched = false;
const storage = storageStub({
getFile: () => {
storageTouched = true;
return null;
},
});

const result = writeFile(storage, {
path: `/reddit/subreddits/localllama/posts/${segment}.json`,
ifMatch: "*",
content: "{}",
});

expect(result).toEqual({ ok: false, error: "invalid_input" });
expect(storageTouched).toBe(false);
});

it("allows a write with normal-length path segments to proceed past the guard", () => {
const storage = storageStub();
const result = writeFile(storage, {
path: "/reddit/subreddits/localllama/posts/a-normal-title__abc123.json",
ifMatch: "0",
content: "{}",
});

expect(result.ok).toBe(true);
});
});
24 changes: 24 additions & 0 deletions packages/core/src/files.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,9 @@ export function writeFile(storage: StorageAdapter, req: WriteFileRequest): Write
}

const path = normalizePath(req.path);
if (hasOversizedPathSegment(path)) {
return { ok: false, error: "invalid_input" };
}
const ifMatch = normalizeIfMatch(req.ifMatch);
if (!ifMatch) {
return { ok: false, error: "missing_precondition" };
Expand Down Expand Up @@ -274,6 +277,27 @@ export function deleteFile(storage: StorageAdapter, req: DeleteFileRequest): Del
export const DEFAULT_CONTENT_TYPE = "text/markdown";
export const MAX_FILE_BYTES = 10 * 1024 * 1024;

// Local mounts materialize each path segment as a real filesystem entry, and
// most filesystems (ext4, APFS, ...) reject names over 255 bytes with
// ENAMETOOLONG -- worse once a mount's own atomic-write suffix (e.g.
// `.tmp-<random>`) is appended to the leaf segment. A provider adapter that
// builds a path segment from unbounded user content (a long title, etc.) can
// produce a canonical path no mount can ever materialize, which then fails
// forever on every sync cycle since nothing else revisits an already-written
// path. Reject that at write time instead of a bounded content-derived slug
// having to be probabilistically clean everywhere it's used.
export const MAX_PATH_SEGMENT_BYTES = 200;

function encodedByteLength(value: string): number {
return new TextEncoder().encode(value).length;
}

export function hasOversizedPathSegment(path: string): boolean {
return path
.split("/")
.some((segment) => segment.length > 0 && encodedByteLength(segment) > MAX_PATH_SEGMENT_BYTES);
}

function normalizeEncoding(value?: string): "utf-8" | "base64" | null {
const encoding = value?.trim().toLowerCase() ?? "";
if (!encoding || encoding === "utf-8" || encoding === "utf8") {
Expand Down
27 changes: 27 additions & 0 deletions packages/core/src/webhooks.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,33 @@ describe("webhook Slack path canonicalization", () => {
"/slack/channels/C123/messages/1711111111_000100/meta.json",
);
});

it("rejects an envelope write whose path has an oversized segment instead of writing it", () => {
const storage = new MemoryStorage();
const oversizedSlug = "a".repeat(250);

const queued = ingestWebhook(storage, {
provider: "reddit",
eventType: "file.created",
path: `/reddit/subreddits/localllama/posts/${oversizedSlug}__1uvwn9q.json`,
deliveryId: "delivery_1",
timestamp: "2026-07-15T09:45:00.000Z",
correlationId: "corr_1",
}, {
generateEnvelopeId: () => "env_1",
coalesceWindowMs: 10_000,
});
expect(queued.status).toBe("queued");

const envelope = storage.envelopes.get("env_1");
if (!envelope) throw new Error("envelope not queued");

const result = applyWebhookEnvelope(storage, envelope);

expect(result.status).toBe("rejected");
expect(result.reason).toBe("path_too_long");
expect(storage.files.size).toBe(0);
});
});

function fileRow(path: string, overrides: Partial<FileRow> = {}): FileRow {
Expand Down
12 changes: 11 additions & 1 deletion packages/core/src/webhooks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import type {
Paginated,
FileSemantics,
} from "./storage.js";
import { normalizePath, DEFAULT_CONTENT_TYPE, MAX_FILE_BYTES, encodedSize } from "./files.js";
import { normalizePath, DEFAULT_CONTENT_TYPE, MAX_FILE_BYTES, encodedSize, hasOversizedPathSegment } from "./files.js";

export interface IngestWebhookInput {
provider: string;
Expand Down Expand Up @@ -397,6 +397,16 @@ export function applyWebhookEnvelope(
};
}

if (hasOversizedPathSegment(event.path)) {
return {
status: "rejected",
eventType: event.type,
path: event.path,
revision: null,
reason: "path_too_long",
};
}

const content =
typeof event.content === "string"
? event.content
Expand Down
Loading