feat: use remote dev for r2 cache population

Use Wrangler remote dev for remote R2 cache population, ensure the target bucket exists before uploading cache entries, and isolate the helper worker from project config discovery.

Co-authored-by: Victor Berchet <victor@suumit.com>

Co-authored-by: Isaac Rowntree <isaac@rowntree.me>

Author: edmundhung

.changeset/fix-r2-cache-upload.md

@@ -0,0 +1,7 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+Use remote dev for R2 cache population
+
+Using remote dev is not subject the Cloudflare API rate limit of 1,200 requests per 5 minutes that caused failures for large applications with thousands of prerendered pages.

packages/cloudflare/src/cli/commands/deploy.ts

@@ -30,19 +30,24 @@ export async function deployCommand(args: WithWranglerArgs<{ cacheChunkSize?: nu
 
 	const envVars = await getEnvFromPlatformProxy(config, buildOpts);
 
-	await populateCache(
-		buildOpts,
-		config,
-		wranglerConfig,
-		{
-			target: "remote",
-			environment: args.env,
-			wranglerConfigPath: args.wranglerConfigPath,
-			cacheChunkSize: args.cacheChunkSize,
-			shouldUsePreviewId: false,
-		},
-		envVars
-	);
+	try {
+		await populateCache(
+			buildOpts,
+			config,
+			wranglerConfig,
+			{
+				target: "remote",
+				environment: args.env,
+				wranglerConfigPath: args.wranglerConfigPath,
+				cacheChunkSize: args.cacheChunkSize,
+				shouldUsePreviewId: false,
+			},
+			envVars
+		);
+	} catch (error) {
+		logger.error(error instanceof Error ? error.message : String(error));
+		process.exit(1);
+	}
 
 	const deploymentMapping = await getDeploymentMapping(buildOpts, config, envVars);
 

packages/cloudflare/src/cli/commands/populate-cache.spec.ts

@@ -2,10 +2,15 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import path from "node:path";
 
 import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
+import { OpenNextConfig } from "@opennextjs/aws/types/open-next.js";
 import mockFs from "mock-fs";
 import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest";
+import type { Unstable_Config as WranglerConfig } from "wrangler";
+import { unstable_startWorker } from "wrangler";
 
-import { getCacheAssets, populateCache } from "./populate-cache.js";
+import { ensureR2Bucket } from "../utils/ensure-r2-bucket.js";
+import { getCacheAssets, populateCache, PopulateCacheOptions } from "./populate-cache.js";
+import { WorkerEnvVar } from "./utils/helpers.js";
 
 describe("getCacheAssets", () => {
 	beforeAll(() => {
@@ -78,7 +83,37 @@ vi.mock("./utils/helpers.js", () => ({
 	quoteShellMeta: vi.fn((s) => s),
 }));
 
+vi.mock("../utils/ensure-r2-bucket.js");
+vi.mock("wrangler");
+
 describe("populateCache", () => {
+	// @ts-expect-error - Partial mock of OpenNextConfig for testing
+	const buildOptions: BuildOptions = {
+		appPath: "/test/app",
+		outputDir: "/test/output",
+	};
+	const config: OpenNextConfig = {
+		default: {
+			override: {
+				// @ts-expect-error - Use R2 incremental cache
+				incrementalCache: "cf-r2-incremental-cache",
+			},
+		},
+	};
+	// @ts-expect-error - Partial mock of WranglerConfig for testing
+	const wranglerConfig: WranglerConfig = {
+		r2_buckets: [
+			{
+				binding: "NEXT_INC_CACHE_R2_BUCKET",
+				bucket_name: "test-bucket",
+				preview_bucket_name: "preview-bucket",
+				jurisdiction: "eu",
+			},
+		],
+	};
+	// @ts-expect-error - Use partial WorkerEnvVar for testing
+	const envVars: WorkerEnvVar = {};
+
 	const setupMockFileSystem = () => {
 		mockFs({
 			"/test/output": {
@@ -95,85 +130,103 @@ describe("populateCache", () => {
 		});
 	};
 
-	describe.each([{ target: "local" as const }, { target: "remote" as const }])(
-		"R2 incremental cache",
-		({ target }) => {
-			afterEach(() => {
-				mockFs.restore();
-			});
+	describe("R2 incremental cache", () => {
+		afterEach(() => {
+			vi.resetAllMocks();
+			mockFs.restore();
+		});
 
-			test(target, async () => {
-				const { runWrangler } = await import("./utils/run-wrangler.js");
+		test.each<PopulateCacheOptions>([
+			{ target: "local", shouldUsePreviewId: false },
+			{ target: "remote", shouldUsePreviewId: false },
+			{ target: "remote", shouldUsePreviewId: true },
+		])(
+			`$target (shouldUsePreviewId: $shouldUsePreviewId) - starts worker and sends individual cache entries via FormData`,
+			async (populateCacheOptions) => {
+				const bucketName =
+					populateCacheOptions.target === "remote" && populateCacheOptions.shouldUsePreviewId
+						? "preview-bucket"
+						: "test-bucket";
+				const mockWorkerDispose = vi.fn();
 
 				setupMockFileSystem();
-				vi.mocked(runWrangler).mockClear();
-
-				await populateCache(
-					{
-						outputDir: "/test/output",
-					} as BuildOptions,
-					{
-						default: {
-							override: {
-								incrementalCache: "cf-r2-incremental-cache",
-							},
-						},
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{
-						r2_buckets: [
-							{
-								binding: "NEXT_INC_CACHE_R2_BUCKET",
-								bucket_name: "test-bucket",
-							},
-						],
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{ target, shouldUsePreviewId: false },
-					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+				// @ts-expect-error - Mock unstable_startWorker to return a mock worker instance
+				vi.mocked(unstable_startWorker).mockResolvedValueOnce({
+					ready: Promise.resolve(),
+					url: Promise.resolve(new URL("http://localhost:12345")),
+					dispose: mockWorkerDispose,
+				});
+				vi.mocked(ensureR2Bucket).mockResolvedValueOnce({ success: true, bucketName });
+
+				// Mock fetch to return a successful response for each individual entry.
+				const fetchMock = vi.spyOn(global, "fetch").mockResolvedValue(
+					new Response(JSON.stringify({ success: true }), {
+						status: 200,
+						headers: { "Content-Type": "application/json" },
+					})
 				);
 
-				expect(runWrangler).toHaveBeenCalledWith(
-					expect.anything(),
-					expect.arrayContaining(["r2 bulk put", "test-bucket"]),
-					expect.objectContaining({ target })
-				);
-			});
-
-			test(`${target} using jurisdiction`, async () => {
-				const { runWrangler } = await import("./utils/run-wrangler.js");
+				await populateCache(buildOptions, config, wranglerConfig, populateCacheOptions, envVars);
 
-				setupMockFileSystem();
-				vi.mocked(runWrangler).mockClear();
-
-				await populateCache(
-					{
-						outputDir: "/test/output",
-					} as BuildOptions,
-					{
-						default: {
-							override: {
-								incrementalCache: "cf-r2-incremental-cache",
-							},
-						},
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{
-						r2_buckets: [
-							{
-								binding: "NEXT_INC_CACHE_R2_BUCKET",
-								bucket_name: "test-bucket",
+				expect(unstable_startWorker).toHaveBeenCalledWith(
+					expect.objectContaining({
+						bindings: expect.objectContaining({
+							R2: expect.objectContaining({
+								type: "r2_bucket",
+								bucket_name: bucketName,
 								jurisdiction: "eu",
-							},
-						],
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{ target, shouldUsePreviewId: false },
-					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+							}),
+						}),
+						dev: expect.objectContaining({
+							remote: populateCacheOptions.target === "remote",
+						}),
+					})
 				);
 
-				expect(runWrangler).toHaveBeenCalledWith(
-					expect.anything(),
-					expect.arrayContaining(["r2 bulk put", "test-bucket", "--jurisdiction eu"]),
-					expect.objectContaining({ target })
-				);
-			});
-		}
-	);
+				if (populateCacheOptions.target === "remote") {
+					expect(ensureR2Bucket).toHaveBeenCalledWith("/test/app", bucketName, "eu");
+				} else {
+					expect(ensureR2Bucket).not.toHaveBeenCalled();
+				}
+
+				expect(fetchMock).toBeCalled();
+
+				for (const [input, init] of fetchMock.mock.calls) {
+					expect(input).toBe("http://localhost:12345/populate");
+					expect(init?.method).toBe("POST");
+
+					const formData = init?.body;
+					if (formData instanceof FormData) {
+						// Verify the body is FormData containing key and value fields.
+						expect(formData.get("key")).toBeTypeOf("string");
+						expect(formData.get("value")).toBeTypeOf("string");
+					} else {
+						expect.unreachable("Expected request body to be FormData");
+					}
+				}
+
+				// Verify worker was disposed after sending entries.
+				expect(mockWorkerDispose).toHaveBeenCalled();
+			}
+		);
+
+		test("remote - exits when bucket provisioning fails", async () => {
+			setupMockFileSystem();
+			vi.mocked(ensureR2Bucket).mockResolvedValueOnce({ success: false });
+
+			const result = populateCache(
+				buildOptions,
+				config,
+				wranglerConfig,
+				{ target: "remote", shouldUsePreviewId: false },
+				envVars
+			);
+
+			await expect(result).rejects.toThrow(
+				'Failed to provision remote R2 bucket "test-bucket" for binding "NEXT_INC_CACHE_R2_BUCKET".'
+			);
+
+			expect(unstable_startWorker).not.toHaveBeenCalled();
+		});
+	});
 });