Skip to content

Commit b0c7880

Browse files
khoinguyenpham04khoinguyenpham04
authored
fix: make media uploads reliable across storage backends (#2273)
* fix: support streamed media uploads * fix: harden streamed media uploads * fix: prevent media cleanup confirmation race * fix: harden media upload integrity * fix: close media upload integrity gaps * test: synchronize media upload failure assertion * chore: tighten media upload comments * fix: preserve media upload retry integrity * fix: reject stale media confirmations
1 parent 81e86f6 commit b0c7880

32 files changed

Lines changed: 2766 additions & 159 deletions

File tree

‎.changeset/stream-media-uploads.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
"emdash": patch
3+
"@emdash-cms/admin": patch
4+
---
5+
6+
Fixes media uploads with native R2 storage, keeps client-side hashing optional, and prevents deduplication from returning media with a different type or size.
7+
Images larger than 8 MiB skip server-generated placeholders in signed and streamed upload flows.

‎docs/src/content/docs/contributing/architecture.mdx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -245,12 +245,12 @@ Portable Text fields edit in TipTap (ProseMirror). Content is converted at the l
245245

246246
### Signed uploads
247247

248-
Media uploads bypass Worker body-size limits with direct-to-storage signed URLs:
248+
Media uploads use direct-to-storage signed URLs when the adapter supports them and a same-origin streaming endpoint otherwise:
249249

250250
<Steps>
251251

252252
1. The client requests an upload URL (`POST /api/media/upload-url`).
253-
2. The client uploads directly to the signed URL (R2 or S3).
253+
2. The client uploads to the returned target. S3-compatible adapters can return a signed URL that bypasses Worker body-size limits; native R2 bindings and local storage return an EmDash streaming endpoint.
254254
3. The client confirms (`POST /api/media/:id/confirm`).
255255
4. The server extracts metadata (dimensions, MIME type).
256256

‎packages/admin/src/lib/api/media.ts‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,6 @@ import {
1313
type FindManyResult,
1414
} from "./client.js";
1515

16-
/**
17-
* Maximum length of the media filename search term. Mirrors the server-side
18-
* zod schema (`q: z.string().trim().min(1).max(200)`); keep in sync.
19-
*/
2016
export const MEDIA_SEARCH_MAX_LENGTH = 200;
2117

2218
/** Trim and clamp a search term to the server-accepted range. */
@@ -103,22 +99,47 @@ interface UploadUrlResponse {
10399
expiresAt: string;
104100
}
105101

102+
interface ExistingMediaResponse {
103+
existing: true;
104+
mediaId: string;
105+
storageKey: string;
106+
url: string;
107+
}
108+
109+
const MAX_CLIENT_HASH_BYTES = 8 * 1024 * 1024;
110+
111+
async function computeContentHash(file: File): Promise<string | undefined> {
112+
const subtle = globalThis.crypto?.subtle;
113+
if (!subtle || file.size === 0 || file.size > MAX_CLIENT_HASH_BYTES) return undefined;
114+
try {
115+
const hash = await subtle.digest("SHA-1", await file.arrayBuffer());
116+
const hex = Array.from(new Uint8Array(hash), (byte) => byte.toString(16).padStart(2, "0")).join(
117+
"",
118+
);
119+
return `sha1:${hex}`;
120+
} catch {
121+
return undefined;
122+
}
123+
}
124+
106125
/**
107126
* Try to get a signed upload URL
108127
* Returns null if signed URLs are not supported (e.g., local storage)
109128
*/
110129
async function getUploadUrl(
111130
file: File,
112131
opts?: { fieldId?: string },
113-
): Promise<UploadUrlResponse | null> {
132+
): Promise<UploadUrlResponse | ExistingMediaResponse | null> {
114133
try {
134+
const contentHash = await computeContentHash(file);
115135
const response = await apiFetch(`${API_BASE}/media/upload-url`, {
116136
method: "POST",
117137
headers: { "Content-Type": "application/json" },
118138
body: JSON.stringify({
119139
filename: file.name,
120140
contentType: file.type,
121141
size: file.size,
142+
...(contentHash ? { contentHash } : {}),
122143
...(opts?.fieldId ? { fieldId: opts.fieldId } : {}),
123144
}),
124145
});
@@ -128,7 +149,10 @@ async function getUploadUrl(
128149
return null;
129150
}
130151

131-
return parseApiResponse<UploadUrlResponse>(response, i18n._(msg`Failed to get upload URL`));
152+
return parseApiResponse<UploadUrlResponse | ExistingMediaResponse>(
153+
response,
154+
i18n._(msg`Failed to get upload URL`),
155+
);
132156
} catch (error) {
133157
// If the endpoint doesn't exist, fall back to direct upload
134158
if (error instanceof TypeError && error.message.includes("fetch")) {
@@ -234,6 +258,9 @@ export async function uploadMedia(file: File, opts?: { fieldId?: string }): Prom
234258
// Signed URLs not supported, use direct upload
235259
return uploadMediaDirect(file, opts);
236260
}
261+
if ("existing" in uploadInfo) {
262+
return fetchMediaItem(uploadInfo.mediaId);
263+
}
237264

238265
// Upload directly to storage via signed URL
239266
await uploadToSignedUrl(file, uploadInfo);

‎packages/admin/src/router.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1341,7 +1341,9 @@ function MediaPage() {
13411341
isLoading={isLoading || isFetchingNextPage}
13421342
hasMore={!!hasNextPage}
13431343
onLoadMore={() => void fetchNextPage()}
1344-
onUpload={(file) => uploadMutation.mutate(file)}
1344+
onUpload={async (file) => {
1345+
await uploadMutation.mutateAsync(file);
1346+
}}
13451347
onLocalSearchChange={setSearch}
13461348
onLocalMimeFilterChange={setMimeFilter}
13471349
/>

‎packages/admin/tests/lib/media-upload.test.ts‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
import { afterEach, expect, it, vi } from "vitest";
2+
3+
import { uploadMedia } from "../../src/lib/api/media.js";
4+
5+
afterEach(() => {
6+
vi.restoreAllMocks();
7+
vi.unstubAllGlobals();
8+
});
9+
10+
it("deduplicates uploads using the file content hash", async () => {
11+
let uploadUrlBody: Record<string, unknown> | undefined;
12+
const fetch = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
13+
const url =
14+
typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
15+
if (url === "/_emdash/api/media/upload-url") {
16+
if (typeof init?.body !== "string") throw new TypeError("Expected a JSON request body");
17+
uploadUrlBody = JSON.parse(init.body) as Record<string, unknown>;
18+
return Response.json({
19+
success: true,
20+
data: {
21+
existing: true,
22+
mediaId: "existing-media",
23+
storageKey: "existing.pdf",
24+
url: "/_emdash/api/media/file/existing.pdf",
25+
},
26+
});
27+
}
28+
if (url === "/_emdash/api/media/existing-media") {
29+
return Response.json({
30+
success: true,
31+
data: {
32+
item: {
33+
id: "existing-media",
34+
filename: "existing.pdf",
35+
mimeType: "application/pdf",
36+
url: "/_emdash/api/media/file/existing.pdf",
37+
storageKey: "existing.pdf",
38+
size: 3,
39+
createdAt: "2026-01-01T00:00:00.000Z",
40+
},
41+
},
42+
});
43+
}
44+
return new Response(null, { status: 500 });
45+
});
46+
const file = new File([new Uint8Array([97, 98, 99])], "document.pdf", {
47+
type: "application/pdf",
48+
});
49+
50+
const item = await uploadMedia(file);
51+
52+
expect(uploadUrlBody?.contentHash).toBe("sha1:a9993e364706816aba3e25717850c26c9cd0d89d");
53+
expect(item.id).toBe("existing-media");
54+
expect(fetch).toHaveBeenCalledTimes(2);
55+
});
56+
57+
it("uploads without deduplication when Web Crypto is unavailable", async () => {
58+
vi.stubGlobal("crypto", {});
59+
let uploadUrlBody: Record<string, unknown> | undefined;
60+
vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
61+
const url =
62+
typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
63+
if (url === "/_emdash/api/media/upload-url") {
64+
if (typeof init?.body !== "string") throw new TypeError("Expected a JSON request body");
65+
uploadUrlBody = JSON.parse(init.body) as Record<string, unknown>;
66+
return Response.json({
67+
success: true,
68+
data: {
69+
uploadUrl: "/_emdash/api/media/new-media/upload",
70+
method: "PUT",
71+
headers: { "Content-Type": "application/pdf" },
72+
mediaId: "new-media",
73+
storageKey: "new.pdf",
74+
expiresAt: "2026-01-01T01:00:00.000Z",
75+
},
76+
});
77+
}
78+
if (url === "/_emdash/api/media/new-media/upload") {
79+
return Response.json({ success: true, data: { uploaded: true, size: 3 } });
80+
}
81+
if (url === "/_emdash/api/media/new-media/confirm") {
82+
return Response.json({
83+
success: true,
84+
data: {
85+
item: {
86+
id: "new-media",
87+
filename: "new.pdf",
88+
mimeType: "application/pdf",
89+
url: "/_emdash/api/media/file/new.pdf",
90+
storageKey: "new.pdf",
91+
size: 3,
92+
createdAt: "2026-01-01T00:00:00.000Z",
93+
},
94+
},
95+
});
96+
}
97+
return new Response(null, { status: 500 });
98+
});
99+
const file = new File([new Uint8Array([1, 2, 3])], "new.pdf", {
100+
type: "application/pdf",
101+
});
102+
103+
const item = await uploadMedia(file);
104+
105+
expect(uploadUrlBody).not.toHaveProperty("contentHash");
106+
expect(item.id).toBe("new-media");
107+
});
108+
109+
it("uploads without deduplication when content hashing fails", async () => {
110+
vi.stubGlobal("crypto", {
111+
subtle: {
112+
digest: vi.fn().mockRejectedValue(new Error("SHA-1 unavailable")),
113+
},
114+
});
115+
let uploadUrlBody: Record<string, unknown> | undefined;
116+
const fetch = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
117+
const url =
118+
typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
119+
if (url === "/_emdash/api/media/upload-url") {
120+
if (typeof init?.body !== "string") throw new TypeError("Expected a JSON request body");
121+
uploadUrlBody = JSON.parse(init.body) as Record<string, unknown>;
122+
return new Response(null, { status: 501 });
123+
}
124+
if (url === "/_emdash/api/media") {
125+
return Response.json({
126+
success: true,
127+
data: {
128+
item: {
129+
id: "new-media",
130+
filename: "new.pdf",
131+
mimeType: "application/pdf",
132+
url: "/_emdash/api/media/file/new.pdf",
133+
storageKey: "new.pdf",
134+
size: 3,
135+
createdAt: "2026-01-01T00:00:00.000Z",
136+
},
137+
},
138+
});
139+
}
140+
return new Response(null, { status: 500 });
141+
});
142+
const file = new File([new Uint8Array([1, 2, 3])], "new.pdf", {
143+
type: "application/pdf",
144+
});
145+
146+
const item = await uploadMedia(file);
147+
148+
expect(uploadUrlBody).not.toHaveProperty("contentHash");
149+
expect(item.id).toBe("new-media");
150+
expect(fetch).toHaveBeenCalledTimes(2);
151+
});
152+
153+
it("does not deduplicate empty files by their shared hash", async () => {
154+
let uploadUrlBody: Record<string, unknown> | undefined;
155+
vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
156+
const url =
157+
typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
158+
if (url === "/_emdash/api/media/upload-url") {
159+
if (typeof init?.body !== "string") throw new TypeError("Expected a JSON request body");
160+
uploadUrlBody = JSON.parse(init.body) as Record<string, unknown>;
161+
return Response.json({
162+
success: true,
163+
data: {
164+
uploadUrl: "/_emdash/api/media/empty-media/upload",
165+
method: "PUT",
166+
headers: { "Content-Type": "application/pdf" },
167+
mediaId: "empty-media",
168+
storageKey: "empty.pdf",
169+
expiresAt: "2026-01-01T01:00:00.000Z",
170+
},
171+
});
172+
}
173+
if (url === "/_emdash/api/media/empty-media/upload") {
174+
return Response.json({ success: true, data: { uploaded: true, size: 0 } });
175+
}
176+
if (url === "/_emdash/api/media/empty-media/confirm") {
177+
return Response.json({
178+
success: true,
179+
data: {
180+
item: {
181+
id: "empty-media",
182+
filename: "empty.pdf",
183+
mimeType: "application/pdf",
184+
url: "/_emdash/api/media/file/empty.pdf",
185+
storageKey: "empty.pdf",
186+
size: 0,
187+
createdAt: "2026-01-01T00:00:00.000Z",
188+
},
189+
},
190+
});
191+
}
192+
return new Response(null, { status: 500 });
193+
});
194+
195+
await uploadMedia(new File([], "empty.pdf", { type: "application/pdf" }));
196+
197+
expect(uploadUrlBody).not.toHaveProperty("contentHash");
198+
});

0 commit comments

Comments
 (0)