fix: Use worker binding for R2 cache uploads to avoid API rate limits

.changeset/fix-r2-cache-upload.md

@@ -0,0 +1,18 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+fix: Use remote R2 binding for cache population to avoid API rate limits
+
+For deployments with prerendered pages, the R2 incremental cache is now populated
+using `unstable_startWorker` with a remote R2 binding instead of `wrangler r2 bulk put`.
+This bypasses the Cloudflare API rate limit of 1,200 requests per 5 minutes that caused
+failures for large applications with thousands of prerendered pages.
+
+The new approach:
+1. Starts a local worker via `unstable_startWorker` with the R2 binding configured programmatically
+2. Sends cache entries to the local worker a few at a time for low memory usage
+3. The worker writes entries to R2 via the binding (no API rate limits)
+4. No deployment, temp config files, or authentication tokens required
+
+Closes #1088

packages/cloudflare/src/cli/commands/populate-cache.spec.ts

@@ -78,6 +78,20 @@ vi.mock("./utils/helpers.js", () => ({
 	quoteShellMeta: vi.fn((s) => s),
 }));
 
+const mockWorkerFetch = vi.fn();
+const mockWorkerDispose = vi.fn();
+
+vi.mock("wrangler", () => ({
+	unstable_startWorker: vi.fn(() =>
+		Promise.resolve({
+			ready: Promise.resolve(),
+			url: Promise.resolve(new URL("http://localhost:12345")),
+			fetch: mockWorkerFetch,
+			dispose: mockWorkerDispose,
+		})
+	),
+}));
+
 describe("populateCache", () => {
 	const setupMockFileSystem = () => {
 		mockFs({
@@ -100,13 +114,19 @@ describe("populateCache", () => {
 		({ target }) => {
 			afterEach(() => {
 				mockFs.restore();
+				vi.clearAllMocks();
 			});
 
-			test(target, async () => {
-				const { runWrangler } = await import("./utils/run-wrangler.js");
-
+			test(`${target} - starts worker and sends cache entries`, async () => {
 				setupMockFileSystem();
-				vi.mocked(runWrangler).mockClear();
+
+				// Mock fetch to return a successful response for each batch
+				global.fetch = vi.fn().mockResolvedValue(
+					new Response(JSON.stringify({ written: 1, failed: 0 }), {
+						status: 200,
+						headers: { "Content-Type": "application/json" },
+					})
+				);
 
 				await populateCache(
 					{
@@ -131,49 +151,108 @@ describe("populateCache", () => {
 					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
 				);
 
-				expect(runWrangler).toHaveBeenCalledWith(
-					expect.anything(),
-					expect.arrayContaining(["r2 bulk put", "test-bucket"]),
-					expect.objectContaining({ target })
+				const { unstable_startWorker: startWorker } = await import("wrangler");
+				expect(startWorker).toHaveBeenCalledWith(
+					expect.objectContaining({
+						name: "open-next-cache-populate",
+						compatibilityDate: "2026-01-01",
+						bindings: expect.objectContaining({
+							NEXT_INC_CACHE_R2_BUCKET: expect.objectContaining({
+								type: "r2_bucket",
+								bucket_name: "test-bucket",
+								remote: target === "remote",
+							}),
+						}),
+					})
+				);
+
+				// Verify fetch was called with the /populate URL
+				expect(global.fetch).toHaveBeenCalledWith(
+					"http://localhost:12345/populate",
+					expect.objectContaining({ method: "POST" })
 				);
+
+				// Verify worker was disposed
+				expect(mockWorkerDispose).toHaveBeenCalled();
 			});
+		}
+	);
 
-			test(`${target} using jurisdiction`, async () => {
-				const { runWrangler } = await import("./utils/run-wrangler.js");
+	describe("retry on partial failures", () => {
+		afterEach(() => {
+			mockFs.restore();
+			vi.clearAllMocks();
+		});
 
-				setupMockFileSystem();
-				vi.mocked(runWrangler).mockClear();
+		test("retries failed entries from 207 response", async () => {
+			setupMockFileSystem();
 
-				await populateCache(
-					{
-						outputDir: "/test/output",
-					} as BuildOptions,
-					{
-						default: {
-							override: {
-								incrementalCache: "cf-r2-incremental-cache",
-							},
-						},
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{
-						r2_buckets: [
-							{
-								binding: "NEXT_INC_CACHE_R2_BUCKET",
-								bucket_name: "test-bucket",
-								jurisdiction: "eu",
-							},
-						],
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{ target, shouldUsePreviewId: false },
-					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			// First call: partial failure (207), second call: success
+			global.fetch = vi
+				.fn()
+				.mockResolvedValueOnce(
+					new Response(
+						JSON.stringify({
+							written: 0,
+							failed: 1,
+							errors: [
+								"incremental-cache/buildID/abc123.cache: put: Unspecified error (0)",
+							],
+						}),
+						{ status: 207, headers: { "Content-Type": "application/json" } }
+					)
+				)
+				.mockResolvedValue(
+					new Response(JSON.stringify({ written: 1, failed: 0 }), {
+						status: 200,
+						headers: { "Content-Type": "application/json" },
+					})
 				);
 
-				expect(runWrangler).toHaveBeenCalledWith(
-					expect.anything(),
-					expect.arrayContaining(["r2 bulk put", "test-bucket", "--jurisdiction eu"]),
-					expect.objectContaining({ target })
+			await populateCache(
+				{ outputDir: "/test/output" } as BuildOptions,
+				{
+					default: { override: { incrementalCache: "cf-r2-incremental-cache" } },
+				} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{
+					r2_buckets: [{ binding: "NEXT_INC_CACHE_R2_BUCKET", bucket_name: "test-bucket" }],
+				} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{ target: "remote", shouldUsePreviewId: false },
+				{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			// Should have been called at least twice (initial + retry)
+			expect(global.fetch).toHaveBeenCalledTimes(2);
+		});
+
+		test("retries on network errors", async () => {
+			setupMockFileSystem();
+
+			// First call: network error, second call: success
+			global.fetch = vi
+				.fn()
+				.mockRejectedValueOnce(new Error("fetch failed"))
+				.mockResolvedValue(
+					new Response(JSON.stringify({ written: 1, failed: 0 }), {
+						status: 200,
+						headers: { "Content-Type": "application/json" },
+					})
 				);
-			});
-		}
-	);
+
+			await populateCache(
+				{ outputDir: "/test/output" } as BuildOptions,
+				{
+					default: { override: { incrementalCache: "cf-r2-incremental-cache" } },
+				} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{
+					r2_buckets: [{ binding: "NEXT_INC_CACHE_R2_BUCKET", bucket_name: "test-bucket" }],
+				} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{ target: "remote", shouldUsePreviewId: false },
+				{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			// Should have been called at least twice (initial + retry)
+			expect(global.fetch).toHaveBeenCalledTimes(2);
+		});
+	});
 });