diff --git a/.changeset/experimental-preserve-commonjs.md b/.changeset/experimental-preserve-commonjs.md new file mode 100644 index 00000000000..98d15e0326a --- /dev/null +++ b/.changeset/experimental-preserve-commonjs.md @@ -0,0 +1,9 @@ +--- +"@cloudflare/vite-plugin": minor +"@cloudflare/workers-utils": minor +"wrangler": minor +--- + +Experimentally preserve npm CommonJS module boundaries with the new module registry + +When the `new_module_registry` compatibility flag is enabled, Wrangler and the Cloudflare Vite plugin now preserve statically reachable npm CommonJS files as runtime CommonJS modules instead of embedding them in the Worker ES module. This experimental path preserves CommonJS globals, relative `require()` calls, cycles, JSON imports, and ESM default and named interop across builds, local development, and Vite preview. diff --git a/packages/vite-plugin-cloudflare/src/__tests__/build-output.spec.ts b/packages/vite-plugin-cloudflare/src/__tests__/build-output.spec.ts index 98069bb3fb8..d86363ccb20 100644 --- a/packages/vite-plugin-cloudflare/src/__tests__/build-output.spec.ts +++ b/packages/vite-plugin-cloudflare/src/__tests__/build-output.spec.ts @@ -29,4 +29,12 @@ describe("detectModuleType", () => { expect(detectModuleType(filename)).toBe(expected); } ); + + it("prefers a preserved JavaScript module type over the file extension", ({ + expect, + }) => { + expect(detectModuleType("package/index.js", "commonjs")).toBe("cjs"); + expect(detectModuleType("package/data.json", "commonjs")).toBe("cjs"); + expect(detectModuleType("package/index.cjs", "esmodule")).toBe("esm"); + }); }); diff --git a/packages/vite-plugin-cloudflare/src/__tests__/commonjs-module-registry.spec.ts b/packages/vite-plugin-cloudflare/src/__tests__/commonjs-module-registry.spec.ts new file mode 100644 index 00000000000..fbc08a0a8ba --- /dev/null +++ b/packages/vite-plugin-cloudflare/src/__tests__/commonjs-module-registry.spec.ts @@ -0,0 +1,20 @@ +import { describe, it } from "vitest"; +import { + getExperimentalCommonJsModuleName, + isExperimentalCommonJsModuleReference, +} from "../plugins/commonjs-module-registry"; + +describe("experimental CommonJS module references", () => { + it("finds direct and workerd file URL references", ({ expect }) => { + const emittedName = "__cloudflare_cjs__/abc/package/index.js"; + const reference = `__CLOUDFLARE_CJS_MODULE__/worker/${emittedName}`; + + expect(getExperimentalCommonJsModuleName(reference)).toBe(emittedName); + expect( + getExperimentalCommonJsModuleName(`file:///bundle/${reference}`) + ).toBe(emittedName); + expect(isExperimentalCommonJsModuleReference("ordinary-package")).toBe( + false + ); + }); +}); diff --git a/packages/vite-plugin-cloudflare/src/__tests__/get-modules-from-manifest.spec.ts b/packages/vite-plugin-cloudflare/src/__tests__/get-modules-from-manifest.spec.ts index 9596d1a79ed..8f201ac2e51 100644 --- a/packages/vite-plugin-cloudflare/src/__tests__/get-modules-from-manifest.spec.ts +++ b/packages/vite-plugin-cloudflare/src/__tests__/get-modules-from-manifest.spec.ts @@ -1,5 +1,12 @@ -import { describe, test } from "vitest"; -import { getModulesFromManifest } from "../miniflare-options"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { removeDirSync } from "@cloudflare/workers-utils"; +import { describe, onTestFinished, test } from "vitest"; +import { + getModulesFromManifest, + getPreviewModules, +} from "../miniflare-options"; describe("getModulesFromManifest", () => { test("hoists `mainModule` to index 0 even when it is not first in the manifest", ({ @@ -100,3 +107,28 @@ describe("getModulesFromManifest", () => { ).toThrow(/`mainModule` "missing\.js" is missing from `modules`/); }); }); + +describe("getPreviewModules", () => { + test("uses the first matching rule when globs overlap", ({ expect }) => { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), "vite-preview-modules-") + ); + onTestFinished(() => removeDirSync(root)); + fs.mkdirSync(path.join(root, "preserved")); + fs.writeFileSync(path.join(root, "index.js"), "export default {};"); + fs.writeFileSync( + path.join(root, "preserved", "module.js"), + "module.exports = {};" + ); + + const result = getPreviewModules(path.join(root, "index.js"), [ + { type: "CommonJS", include: ["preserved/module.js"] }, + { type: "ESModule", include: ["**/*.js"] }, + ]); + + expect(result.modules).toEqual([ + { type: "ESModule", path: "index.js" }, + { type: "CommonJS", path: "preserved/module.js" }, + ]); + }); +}); diff --git a/packages/vite-plugin-cloudflare/src/__tests__/output-config.spec.ts b/packages/vite-plugin-cloudflare/src/__tests__/output-config.spec.ts index 4221cfb211b..d092a5f7581 100644 --- a/packages/vite-plugin-cloudflare/src/__tests__/output-config.spec.ts +++ b/packages/vite-plugin-cloudflare/src/__tests__/output-config.spec.ts @@ -148,4 +148,32 @@ describe("getWorkerOutputConfig", () => { }, ]); }); + + test("declares preserved CommonJS modules before the broad ESM rule", ({ + expect, + }) => { + const root = createRoot(); + const outputConfig = getOutputConfig({ + inputWorkerConfig: workerConfig(root), + workerOutputDirectory: "dist/api_worker", + resolvedViteConfig: resolvedViteConfig(root), + entryFileName: "index.js", + includeAssets: false, + commonJsModuleNames: [ + "__cloudflare_cjs__/abc/package/index.js", + "__cloudflare_cjs__/abc/package/data.json", + ], + }); + + expect(outputConfig.rules).toEqual([ + { + type: "CommonJS", + globs: [ + "__cloudflare_cjs__/abc/package/index.js", + "__cloudflare_cjs__/abc/package/data.json", + ], + }, + { type: "ESModule", globs: ["**/*.js", "**/*.mjs"] }, + ]); + }); }); diff --git a/packages/vite-plugin-cloudflare/src/cloudflare-environment.ts b/packages/vite-plugin-cloudflare/src/cloudflare-environment.ts index f02facc43a9..f922727f0a0 100644 --- a/packages/vite-plugin-cloudflare/src/cloudflare-environment.ts +++ b/packages/vite-plugin-cloudflare/src/cloudflare-environment.ts @@ -4,6 +4,7 @@ import { CoreHeaders } from "miniflare"; import * as vite from "vite"; import { nodeBuiltinsRE } from "./nodejs-compat"; import { additionalModuleRE } from "./plugins/additional-modules"; +import { isExperimentalCommonJsModuleReference } from "./plugins/commonjs-module-registry"; import { ENVIRONMENT_NAME_HEADER, GET_EXPORT_TYPES_PATH, @@ -167,7 +168,10 @@ export class CloudflareDevEnvironment extends vite.DevEnvironment { options?: FetchFunctionOptions ): Promise { // Additional modules (CompiledWasm, Data, Text) - if (additionalModuleRE.test(id)) { + if ( + additionalModuleRE.test(id) || + isExperimentalCommonJsModuleReference(id) + ) { return { externalize: id, type: "module", @@ -222,6 +226,9 @@ export function createCloudflareEnvironmentOptions({ isParentEnvironment: boolean; hasNodeJsCompat: boolean; }): vite.EnvironmentOptions { + const preserveCommonJs = workerConfig.compatibility_flags?.includes( + "new_module_registry" + ); const rollupOptions = isParentEnvironment ? { input: { @@ -288,7 +295,9 @@ export function createCloudflareEnvironmentOptions({ }, optimizeDeps: { // Note: ssr pre-bundling is opt-in and we need to enable it by setting `noDiscovery` to false - noDiscovery: false, + // CommonJS must reach the registry plugin before optimization converts it to ESM. + noDiscovery: preserveCommonJs, + include: preserveCommonJs ? [] : undefined, // Workaround for https://github.com/vitejs/vite/issues/20867 // Longer term solution is to use full-bundle mode rather than `optimizeDeps` ignoreOutdatedRequests: true, diff --git a/packages/vite-plugin-cloudflare/src/index.ts b/packages/vite-plugin-cloudflare/src/index.ts index 1ef33d32a68..ddba3b89aa5 100644 --- a/packages/vite-plugin-cloudflare/src/index.ts +++ b/packages/vite-plugin-cloudflare/src/index.ts @@ -5,6 +5,7 @@ import { PluginContext } from "./context"; import { resolvePluginConfig } from "./plugin-config"; import { additionalModulesPlugin } from "./plugins/additional-modules"; import { buildOutputPlugin } from "./plugins/build-output"; +import { commonJsModuleRegistryPlugin } from "./plugins/commonjs-module-registry"; import { configPlugin } from "./plugins/config"; import { debugPlugin } from "./plugins/debug"; import { devPlugin } from "./plugins/dev"; @@ -115,6 +116,7 @@ export function cloudflare(pluginConfig: PluginConfig = {}): vite.Plugin[] { triggerHandlersPlugin(ctx), virtualModulesPlugin(ctx), virtualClientFallbackPlugin(ctx), + commonJsModuleRegistryPlugin(ctx), outputPlugin, wasmHelperPlugin(ctx), additionalModulesPlugin(ctx), diff --git a/packages/vite-plugin-cloudflare/src/miniflare-options.ts b/packages/vite-plugin-cloudflare/src/miniflare-options.ts index 46ceb863a6f..0984be76801 100644 --- a/packages/vite-plugin-cloudflare/src/miniflare-options.ts +++ b/packages/vite-plugin-cloudflare/src/miniflare-options.ts @@ -37,6 +37,7 @@ import { import { getContainerOptions, getDockerPath } from "./containers"; import { getInputInspectorPort } from "./debug"; import { additionalModuleRE } from "./plugins/additional-modules"; +import { getExperimentalCommonJsModule } from "./plugins/commonjs-module-registry"; import { ENVIRONMENT_NAME_HEADER } from "./shared"; import { checkForNpmUpdate } from "./update-check"; import { @@ -596,6 +597,18 @@ export async function getDevMiniflareOptions( } const rawSpecifier = parsed.rawSpecifier; + const commonJsModule = getExperimentalCommonJsModule( + ctx, + parsed.specifier + ); + if (commonJsModule !== undefined) { + return MiniflareResponse.json({ + name: parsed.specifier, + [commonJsModule.sourceType === "commonjs" + ? "commonJsModule" + : "esModule"]: commonJsModule.transformedSource, + }); + } assert( rawSpecifier, `Unexpected error: no specifier in request to module fallback service.` @@ -645,7 +658,7 @@ export async function getDevMiniflareOptions( }; } -function getPreviewModules( +export function getPreviewModules( main: string, modulesRules: SourcelessWorkerOptions["modulesRules"] ) { @@ -653,6 +666,7 @@ function getPreviewModules( const rootPath = path.dirname(main); const entryPath = path.basename(main); + const seen = new Set([entryPath]); return { rootPath, modules: [ @@ -661,12 +675,18 @@ function getPreviewModules( path: entryPath, } as const, ...modulesRules.flatMap(({ type, include }) => - globSync(include, { cwd: rootPath, ignore: entryPath }).map( - (globPath) => ({ + globSync(include, { cwd: rootPath, ignore: entryPath }) + .filter((globPath) => { + if (seen.has(globPath)) { + return false; + } + seen.add(globPath); + return true; + }) + .map((globPath) => ({ type, path: globPath, - }) - ) + })) ), ], } satisfies Pick; diff --git a/packages/vite-plugin-cloudflare/src/plugins/build-output.ts b/packages/vite-plugin-cloudflare/src/plugins/build-output.ts index 9a12adf8a44..fca27b8c4c9 100644 --- a/packages/vite-plugin-cloudflare/src/plugins/build-output.ts +++ b/packages/vite-plugin-cloudflare/src/plugins/build-output.ts @@ -6,7 +6,9 @@ import { } from "@cloudflare/build-output-utils"; import { MAIN_ENTRY_NAME } from "../cloudflare-environment"; import { createPlugin } from "../utils"; +import { getExperimentalCommonJsModuleTypes } from "./commonjs-module-registry"; import type { ModuleType } from "@cloudflare/config"; +import type { ExperimentalJavaScriptSourceType } from "@cloudflare/workers-utils"; /** * Build Output Specification plugin. Replaces `outputConfigPlugin` when @@ -69,6 +71,10 @@ export const buildOutputPlugin = createPlugin("build-output", (ctx) => { } const modules: Record = {}; + const commonJsModuleTypes = getExperimentalCommonJsModuleTypes( + ctx, + this.environment.name + ); for (const fileName of Object.keys(bundle)) { // Skip Vite's own manifest emitted via `build.manifest: true`. if (fileName === ".vite/manifest.json") { @@ -80,7 +86,9 @@ export const buildOutputPlugin = createPlugin("build-output", (ctx) => { if (importedAssetPaths.has(fileName)) { continue; } - modules[fileName] = { type: detectModuleType(fileName) }; + modules[fileName] = { + type: detectModuleType(fileName, commonJsModuleTypes.get(fileName)), + }; } await writeWorkerConfig(ctx.resolvedViteConfig.root, workerNewConfig, { @@ -108,7 +116,13 @@ export const buildOutputPlugin = createPlugin("build-output", (ctx) => { /** * Map a bundle filename to its declared module type. */ -export function detectModuleType(filename: string): ModuleType { +export function detectModuleType( + filename: string, + explicitType?: ExperimentalJavaScriptSourceType +): ModuleType { + if (explicitType !== undefined) { + return explicitType === "commonjs" ? "cjs" : "esm"; + } const ext = path.extname(filename).toLowerCase(); switch (ext) { diff --git a/packages/vite-plugin-cloudflare/src/plugins/commonjs-module-registry.ts b/packages/vite-plugin-cloudflare/src/plugins/commonjs-module-registry.ts new file mode 100644 index 00000000000..81c8be03cab --- /dev/null +++ b/packages/vite-plugin-cloudflare/src/plugins/commonjs-module-registry.ts @@ -0,0 +1,322 @@ +import * as path from "node:path"; +import { + experimental_classifyJavaScriptFile, + experimental_createCommonJsGraph, +} from "@cloudflare/workers-utils"; +import MagicString from "magic-string"; +import * as vite from "vite"; +import { cleanUrl, createPlugin, isRolldown } from "../utils"; +import type { PluginContext } from "../context"; +import type { + ExperimentalCommonJsGraph, + ExperimentalCommonJsGraphBuilder, + ExperimentalCommonJsGraphModule, +} from "@cloudflare/workers-utils"; + +const moduleReferencePrefix = "__CLOUDFLARE_CJS_MODULE__"; +const virtualWrapperPrefix = "\0cloudflare-commonjs-wrapper:"; + +interface EnvironmentState { + builder?: ExperimentalCommonJsGraphBuilder; + modules: Map; + roots: Map>; + wrappers: Map>; +} + +const states = new WeakMap>(); + +function createEnvironmentState(): EnvironmentState { + return { + modules: new Map(), + roots: new Map(), + wrappers: new Map(), + }; +} + +function getEnvironmentState( + ctx: PluginContext, + environmentName: string +): EnvironmentState { + let environmentStates = states.get(ctx); + if (environmentStates === undefined) { + environmentStates = new Map(); + states.set(ctx, environmentStates); + } + let state = environmentStates.get(environmentName); + if (state === undefined) { + state = createEnvironmentState(); + environmentStates.set(environmentName, state); + } + return state; +} + +function resetEnvironmentState( + ctx: PluginContext, + environmentName: string +): EnvironmentState { + const environmentStates = states.get(ctx) ?? new Map(); + states.set(ctx, environmentStates); + const state = createEnvironmentState(); + environmentStates.set(environmentName, state); + return state; +} + +function isBareModuleSpecifier(specifier: string): boolean { + return ( + !specifier.startsWith(".") && + !specifier.startsWith("/") && + !specifier.startsWith("\\") && + !path.isAbsolute(specifier) + ); +} + +function isInNodeModules(filePath: string): boolean { + return cleanUrl(filePath).split(/[\\/]/).includes("node_modules"); +} + +function createModuleReference( + environmentName: string, + emittedName: string +): string { + return `${moduleReferencePrefix}/${encodeURIComponent(environmentName)}/${emittedName}`; +} + +function parseModuleReference(specifier: string): + | { + environmentName: string; + emittedName: string; + } + | undefined { + let pathname = specifier; + try { + pathname = new URL(specifier).pathname; + } catch {} + const marker = `${moduleReferencePrefix}/`; + const markerIndex = pathname.indexOf(marker); + if (markerIndex === -1) { + return; + } + const reference = pathname.slice(markerIndex + marker.length); + const separatorIndex = reference.indexOf("/"); + if (separatorIndex === -1) { + return; + } + return { + environmentName: decodeURIComponent(reference.slice(0, separatorIndex)), + emittedName: decodeURIComponent(reference.slice(separatorIndex + 1)), + }; +} + +export function getExperimentalCommonJsModuleName( + specifier: string +): string | undefined { + return parseModuleReference(specifier)?.emittedName; +} + +export function isExperimentalCommonJsModuleReference( + specifier: string +): boolean { + return parseModuleReference(specifier) !== undefined; +} + +export function getExperimentalCommonJsModule( + ctx: PluginContext, + specifier: string +): ExperimentalCommonJsGraphModule | undefined { + const reference = parseModuleReference(specifier); + if (reference === undefined) { + return; + } + return states + .get(ctx) + ?.get(reference.environmentName) + ?.modules.get(reference.emittedName); +} + +export function getExperimentalCommonJsModuleTypes( + ctx: PluginContext, + environmentName: string +): ReadonlyMap { + return new Map( + [...getEnvironmentState(ctx, environmentName).modules].map( + ([name, module]) => [name, module.sourceType] + ) + ); +} + +function createInteropWrapper( + graph: ExperimentalCommonJsGraph, + environmentName: string +): string { + let binding = "__commonJsModule"; + while (graph.root.namedExports.includes(binding)) { + binding = `_${binding}`; + } + + return [ + `import ${binding} from ${JSON.stringify(createModuleReference(environmentName, graph.root.emittedName))};`, + `export default ${binding};`, + ...graph.root.namedExports.map( + (name) => `export const ${name} = ${binding}.${name};` + ), + ].join("\n"); +} + +/** Preserve npm CommonJS boundaries for workerd's experimental module registry. */ +export const commonJsModuleRegistryPlugin = createPlugin( + "commonjs-module-registry", + (ctx) => ({ + enforce: "pre", + applyToEnvironment(environment) { + return ctx + .getWorkerConfig(environment.name) + ?.compatibility_flags?.includes("new_module_registry"); + }, + configResolved(config) { + for (const [environmentName, environment] of Object.entries( + config.environments + )) { + if ( + ctx + .getWorkerConfig(environmentName) + ?.compatibility_flags?.includes("new_module_registry") + ) { + environment.optimizeDeps.noDiscovery = true; + environment.optimizeDeps.include = []; + } + } + }, + buildStart() { + resetEnvironmentState(ctx, this.environment.name); + }, + async resolveId(source, importer, options) { + if (isExperimentalCommonJsModuleReference(source)) { + return { id: source, external: true }; + } + if (source.startsWith(virtualWrapperPrefix)) { + return source; + } + + const resolved = await this.resolve(source, importer, { + ...options, + skipSelf: true, + }); + if (resolved === null || resolved.external) { + return; + } + const resolvedPath = cleanUrl(resolved.id); + if ( + !path.isAbsolute(resolvedPath) || + ![".js", ".cjs"].includes(path.extname(resolvedPath)) || + (!isBareModuleSpecifier(source) && + !isInNodeModules(importer ?? "") && + !isInNodeModules(resolvedPath)) || + (await experimental_classifyJavaScriptFile(resolvedPath)) !== "commonjs" + ) { + return; + } + + const state = getEnvironmentState(ctx, this.environment.name); + const requireResolver = + isRolldown && this.environment.mode === "build" + ? undefined + : this.environment.config.createResolver({ isRequire: true }); + state.builder ??= experimental_createCommonJsGraph({ + resolve: async (specifier, graphImporter) => { + if (requireResolver !== undefined) { + return requireResolver(specifier, graphImporter); + } + const dependency = await this.resolve(specifier, graphImporter, { + kind: "require-call", + skipSelf: true, + }); + if (dependency === null || dependency.external) { + return; + } + const dependencyPath = cleanUrl(dependency.id); + return path.isAbsolute(dependencyPath) ? dependencyPath : undefined; + }, + }); + let graphPromise = state.roots.get(resolvedPath); + if (graphPromise === undefined) { + graphPromise = state.builder.discover(resolvedPath); + state.roots.set(resolvedPath, graphPromise); + } + const graph = await graphPromise; + for (const module of graph.modules) { + state.modules.set(module.emittedName, module); + this.addWatchFile(module.sourcePath); + } + + const wrapperId = `${virtualWrapperPrefix}${resolvedPath}`; + state.wrappers.set(wrapperId, graphPromise); + return { + id: wrapperId, + moduleSideEffects: resolved.moduleSideEffects, + }; + }, + async load(id) { + const graph = getEnvironmentState( + ctx, + this.environment.name + ).wrappers.get(id); + return graph === undefined + ? undefined + : createInteropWrapper(await graph, this.environment.name); + }, + hotUpdate(options) { + const state = getEnvironmentState(ctx, this.environment.name); + if ( + [...state.modules.values()].some( + (module) => module.sourcePath === options.file + ) + ) { + void options.server.restart(); + return []; + } + }, + renderChunk(code, chunk) { + const referencePattern = new RegExp( + `${moduleReferencePrefix}/[^/"'\\n\\r]+/([^"'\\n\\r]+)`, + "g" + ); + let magicString: MagicString | undefined; + for (const match of code.matchAll(referencePattern)) { + const [reference, emittedName] = match; + if (emittedName === undefined) { + continue; + } + magicString ??= new MagicString(code); + const relativePath = vite.normalizePath( + path.relative(path.dirname(chunk.fileName), emittedName) + ); + magicString.update( + match.index, + match.index + reference.length, + relativePath.startsWith(".") ? relativePath : `./${relativePath}` + ); + } + if (magicString !== undefined) { + return { + code: magicString.toString(), + map: this.environment.config.build.sourcemap + ? magicString.generateMap({ hires: "boundary" }) + : null, + }; + } + }, + generateBundle() { + for (const module of getEnvironmentState( + ctx, + this.environment.name + ).modules.values()) { + this.emitFile({ + type: "asset", + fileName: module.emittedName, + originalFileName: module.sourcePath, + source: module.transformedSource, + }); + } + }, + }) +); diff --git a/packages/vite-plugin-cloudflare/src/plugins/output-config.ts b/packages/vite-plugin-cloudflare/src/plugins/output-config.ts index 5bce1e2fab4..cfd7ddc161c 100644 --- a/packages/vite-plugin-cloudflare/src/plugins/output-config.ts +++ b/packages/vite-plugin-cloudflare/src/plugins/output-config.ts @@ -7,6 +7,7 @@ import { assertIsNotPreview } from "../context"; import { writeDeployConfig } from "../deploy-config"; import { getLocalDevVarsForPreview } from "../dev-vars"; import { createPlugin } from "../utils"; +import { getExperimentalCommonJsModuleTypes } from "./commonjs-module-registry"; import type { ResolvedWorkerConfig } from "../plugin-config"; import type { Unstable_RawConfig } from "wrangler"; @@ -54,6 +55,11 @@ export const outputConfigPlugin = createPlugin("output-config", (ctx) => { resolvedViteConfig: ctx.resolvedViteConfig, entryFileName: entryChunk.fileName, includeAssets: isEntryWorker || isPrerenderWorker, + commonJsModuleNames: [ + ...getExperimentalCommonJsModuleTypes(ctx, this.environment.name), + ] + .filter(([, type]) => type === "commonjs") + .map(([name]) => name), }); // Infer `upload_source_maps` from Vite's `build.sourcemap` if not explicitly set @@ -151,12 +157,14 @@ export function getOutputConfig({ resolvedViteConfig, entryFileName, includeAssets, + commonJsModuleNames = [], }: { inputWorkerConfig: ResolvedWorkerConfig; workerOutputDirectory: string; resolvedViteConfig: vite.ResolvedConfig; entryFileName: string; includeAssets: boolean; + commonJsModuleNames?: string[]; }): Unstable_RawConfig { const sourceConfigDirectory = inputWorkerConfig.configPath ? path.dirname(inputWorkerConfig.configPath) @@ -170,7 +178,12 @@ export function getOutputConfig({ ...inputWorkerConfig, main: entryFileName, no_bundle: true, - rules: [{ type: "ESModule", globs: ["**/*.js", "**/*.mjs"] }], + rules: [ + ...(commonJsModuleNames.length > 0 + ? [{ type: "CommonJS" as const, globs: commonJsModuleNames }] + : []), + { type: "ESModule", globs: ["**/*.js", "**/*.mjs"] }, + ], assets: includeAssets ? { ...inputWorkerConfig.assets, diff --git a/packages/workers-utils/package.json b/packages/workers-utils/package.json index 53f7992ca9a..e5b928269e5 100644 --- a/packages/workers-utils/package.json +++ b/packages/workers-utils/package.json @@ -50,6 +50,8 @@ "type:tests": "tsc -p ./tests/tsconfig.json" }, "dependencies": { + "acorn": "8.16.0", + "cjs-module-lexer": "1.2.3", "undici": "catalog:default" }, "devDependencies": { diff --git a/packages/workers-utils/src/commonjs-module-graph.ts b/packages/workers-utils/src/commonjs-module-graph.ts new file mode 100644 index 00000000000..c8cd194a26a --- /dev/null +++ b/packages/workers-utils/src/commonjs-module-graph.ts @@ -0,0 +1,651 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { isBuiltin } from "node:module"; +import path from "node:path"; +import { Parser } from "acorn"; +import * as cjsModuleLexer from "cjs-module-lexer"; + +export type ExperimentalJavaScriptSourceType = "commonjs" | "esmodule"; + +export interface ExperimentalCommonJsGraphModule { + /** The absolute source file used by the caller's resolver. */ + sourcePath: string; + /** The safe, registry-relative name to use when emitting this module. */ + emittedName: string; + /** Source with local requires rewritten to other emitted module names. */ + transformedSource: string; + /** The module type the caller should use when emitting this source. */ + sourceType: ExperimentalJavaScriptSourceType; + /** Safe binding identifiers detected for a generated ESM interop wrapper. */ + namedExports: string[]; +} + +export interface ExperimentalCommonJsGraph { + root: ExperimentalCommonJsGraphModule; + modules: ExperimentalCommonJsGraphModule[]; +} + +export type ExperimentalCommonJsResolver = ( + specifier: string, + importer: string +) => Promise; + +export interface ExperimentalCommonJsGraphOptions { + /** Resolve with the integration's own CommonJS conditions. */ + resolve: ExperimentalCommonJsResolver; +} + +interface AstNode { + type: string; + start: number; + end: number; + loc?: { start: { line: number; column: number } }; + [key: string]: unknown; +} + +interface RequireCall { + argument: AstNode; + specifier: string; +} + +interface Scope { + parent?: Scope; + kind: "block" | "function"; + bindings: Set; +} + +interface PackageInfo { + name: string; + root: string; +} + +interface ModuleRecord { + module?: ExperimentalCommonJsGraphModule; + dependencies: string[]; + dependenciesBySpecifier: Map; + reexports: string[]; + directNamedExports: string[]; + complete: Promise; +} + +const REGISTRY_ROOT = "__cloudflare_cjs__"; +let lexerInit: Promise | undefined; + +function isNotFoundError(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + "code" in error && + (error as NodeJS.ErrnoException).code === "ENOENT" + ); +} + +async function readPackageJson( + directory: string +): Promise | undefined> { + try { + return JSON.parse( + await readFile(path.join(directory, "package.json"), "utf8") + ) as Record; + } catch (error) { + if (isNotFoundError(error)) { + return undefined; + } + throw error; + } +} + +function parentDirectories(filePath: string): string[] { + const directories: string[] = []; + let directory = path.dirname(filePath); + while (true) { + directories.push(directory); + const parent = path.dirname(directory); + if (parent === directory) { + return directories; + } + directory = parent; + } +} + +/** + * Classifies a resolved JavaScript file using Node's package type semantics. + */ +export async function experimental_classifyJavaScriptFile( + filePath: string +): Promise { + const extension = path.extname(filePath); + if (extension === ".cjs") { + return "commonjs"; + } + if (extension === ".mjs") { + return "esmodule"; + } + if (extension !== ".js") { + throw new Error( + `Cannot classify ${JSON.stringify(filePath)} as JavaScript: expected a .js, .cjs, or .mjs file.` + ); + } + + for (const directory of parentDirectories(filePath)) { + const packageJson = await readPackageJson(directory); + if (packageJson !== undefined) { + return packageJson.type === "module" ? "esmodule" : "commonjs"; + } + } + return "commonjs"; +} + +function isAstNode(value: unknown): value is AstNode { + return ( + typeof value === "object" && + value !== null && + "type" in value && + typeof (value as { type?: unknown }).type === "string" + ); +} + +function forEachChild(node: AstNode, callback: (child: AstNode) => void): void { + for (const [key, value] of Object.entries(node)) { + if (key === "loc") { + continue; + } + if (isAstNode(value)) { + callback(value); + } else if (Array.isArray(value)) { + for (const item of value) { + if (isAstNode(item)) { + callback(item); + } + } + } + } +} + +function addPatternBindings( + pattern: AstNode | null | undefined, + scope: Scope +): void { + if (pattern == null) { + return; + } + if (pattern.type === "Identifier") { + scope.bindings.add(pattern.name as string); + return; + } + if (pattern.type === "Property") { + addPatternBindings(pattern.value as AstNode, scope); + return; + } + if (pattern.type === "RestElement") { + addPatternBindings(pattern.argument as AstNode, scope); + return; + } + if (pattern.type === "AssignmentPattern") { + addPatternBindings(pattern.left as AstNode, scope); + return; + } + if (pattern.type === "ArrayPattern" || pattern.type === "ObjectPattern") { + forEachChild(pattern, (child) => addPatternBindings(child, scope)); + } +} + +function nearestFunctionScope(scope: Scope): Scope { + while (scope.kind !== "function") { + scope = scope.parent as Scope; + } + return scope; +} + +function buildScopeMap(root: AstNode): Map { + const scopes = new Map(); + const rootScope: Scope = { kind: "function", bindings: new Set() }; + + function visit(node: AstNode, scope: Scope): void { + scopes.set(node, scope); + + if ( + node.type === "FunctionDeclaration" || + node.type === "FunctionExpression" || + node.type === "ArrowFunctionExpression" + ) { + const id = node.id as AstNode | undefined; + if (node.type === "FunctionDeclaration" && id?.type === "Identifier") { + scope.bindings.add(id.name as string); + } + const functionScope: Scope = { + parent: scope, + kind: "function", + bindings: new Set(), + }; + if (node.type === "FunctionExpression" && id?.type === "Identifier") { + functionScope.bindings.add(id.name as string); + } + for (const parameter of node.params as AstNode[]) { + addPatternBindings(parameter, functionScope); + visit(parameter, functionScope); + } + visit(node.body as AstNode, functionScope); + return; + } + + if (node.type === "BlockStatement") { + const blockScope: Scope = { + parent: scope, + kind: "block", + bindings: new Set(), + }; + scopes.set(node, blockScope); + for (const statement of node.body as AstNode[]) { + visit(statement, blockScope); + } + return; + } + + if (node.type === "CatchClause") { + const catchScope: Scope = { + parent: scope, + kind: "block", + bindings: new Set(), + }; + scopes.set(node, catchScope); + const parameter = node.param as AstNode | null; + addPatternBindings(parameter, catchScope); + if (parameter !== null) { + visit(parameter, catchScope); + } + visit(node.body as AstNode, catchScope); + return; + } + + if (node.type === "VariableDeclaration") { + const bindingScope = + node.kind === "var" ? nearestFunctionScope(scope) : scope; + for (const declaration of node.declarations as AstNode[]) { + addPatternBindings(declaration.id as AstNode, bindingScope); + visit(declaration, scope); + } + return; + } + + if ( + (node.type === "ClassDeclaration" || node.type === "ImportDeclaration") && + isAstNode(node.id) + ) { + addPatternBindings(node.id, scope); + } + + forEachChild(node, (child) => visit(child, scope)); + } + + visit(root, rootScope); + return scopes; +} + +function isRequireShadowed(scope: Scope): boolean { + for ( + let current: Scope | undefined = scope; + current; + current = current.parent + ) { + if (current.bindings.has("require")) { + return true; + } + } + return false; +} + +function findRequireCalls(sourcePath: string, source: string): RequireCall[] { + const root = Parser.parse(source, { + ecmaVersion: "latest", + sourceType: "script", + allowHashBang: true, + allowReturnOutsideFunction: true, + locations: true, + }) as unknown as AstNode; + const scopes = buildScopeMap(root); + const calls: RequireCall[] = []; + + function visit(node: AstNode): void { + if (node.type === "CallExpression") { + const callee = node.callee as AstNode; + if ( + callee.type === "Identifier" && + callee.name === "require" && + !isRequireShadowed(scopes.get(node) as Scope) + ) { + const args = node.arguments as AstNode[]; + const argument = args[0]; + if ( + args.length !== 1 || + argument?.type !== "Literal" || + typeof argument.value !== "string" + ) { + const location = node.loc?.start; + const suffix = location + ? `:${location.line}:${location.column + 1}` + : ""; + throw new Error( + `Experimental CommonJS graph cannot analyze dynamic require() in ${sourcePath}${suffix}; only require() with one string literal is supported.` + ); + } + calls.push({ + argument, + specifier: argument.value, + }); + } + } + forEachChild(node, visit); + } + + visit(root); + return calls; +} + +function isSafeBindingIdentifier(name: string): boolean { + if (name === "default" || name === "__esModule") { + return false; + } + try { + const root = Parser.parse(`export const ${name} = 0`, { + ecmaVersion: "latest", + sourceType: "module", + }) as unknown as AstNode; + const body = root.body as AstNode[]; + const declaration = body[0]?.declaration as AstNode | undefined; + const declarators = declaration?.declarations as AstNode[] | undefined; + const identifier = declarators?.[0]?.id as AstNode | undefined; + return ( + body.length === 1 && + declaration?.type === "VariableDeclaration" && + declarators?.length === 1 && + identifier?.type === "Identifier" && + identifier.name === name + ); + } catch { + return false; + } +} + +async function parseNamedExports(source: string): Promise<{ + exports: string[]; + reexports: string[]; +}> { + lexerInit ??= cjsModuleLexer.init(); + await lexerInit; + const result = cjsModuleLexer.parse(source); + return { + exports: result.exports.filter(isSafeBindingIdentifier), + reexports: result.reexports, + }; +} + +function toPosixPath(filePath: string): string { + return filePath.replaceAll(path.sep, path.posix.sep); +} + +function validatePackageName(name: string): string[] { + const parts = name.split("/"); + const valid = name.startsWith("@") + ? parts.length === 2 && parts[0].length > 1 && parts[1].length > 0 + : parts.length === 1 && parts[0].length > 0; + if ( + !valid || + parts.some( + (part) => + part === "." || part === ".." || !/^@?[a-zA-Z0-9._~-]+$/.test(part) + ) + ) { + throw new Error( + `Cannot emit experimental CommonJS package with unsafe name ${JSON.stringify(name)}.` + ); + } + return parts; +} + +async function findPackageInfo(filePath: string): Promise { + for (const directory of parentDirectories(filePath)) { + const packageJson = await readPackageJson(directory); + if (typeof packageJson?.name === "string") { + validatePackageName(packageJson.name); + return { name: packageJson.name, root: directory }; + } + } + throw new Error( + `Cannot emit ${JSON.stringify(filePath)} in the experimental CommonJS registry because it is not inside a named npm package.` + ); +} + +function emittedNameFor(filePath: string, packageInfo: PackageInfo): string { + const relativePath = path.relative(packageInfo.root, filePath); + if ( + relativePath === ".." || + relativePath.startsWith(`..${path.sep}`) || + path.isAbsolute(relativePath) + ) { + throw new Error(`CommonJS module escaped package root: ${filePath}`); + } + const instance = createHash("sha256") + .update(toPosixPath(path.resolve(packageInfo.root))) + .digest("hex") + .slice(0, 10); + return path.posix.join( + REGISTRY_ROOT, + instance, + ...validatePackageName(packageInfo.name), + toPosixPath(relativePath) + ); +} + +function rewriteRequires( + source: string, + replacements: Array<{ argument: AstNode; emittedSpecifier: string }> +): string { + for (const replacement of replacements.sort( + (a, b) => b.argument.start - a.argument.start + )) { + source = + source.slice(0, replacement.argument.start) + + JSON.stringify(replacement.emittedSpecifier) + + source.slice(replacement.argument.end); + } + return source; +} + +/** + * A reusable graph builder. Reusing one instance caches modules by resolved path + * across roots while retaining the resolver's integration-specific conditions. + */ +export class ExperimentalCommonJsGraphBuilder { + readonly #resolve: ExperimentalCommonJsResolver; + readonly #records = new Map(); + #discoveryQueue = Promise.resolve(); + + constructor(options: ExperimentalCommonJsGraphOptions) { + this.#resolve = options.resolve; + } + + discover(rootPath: string): Promise { + const discovery = this.#discoveryQueue.then(() => this.#discover(rootPath)); + this.#discoveryQueue = discovery.then( + () => undefined, + () => undefined + ); + return discovery; + } + + async #discover(rootPath: string): Promise { + rootPath = path.resolve(rootPath); + const root = await this.#visit(rootPath, new Set()); + if (root.sourceType !== "commonjs") { + throw new Error( + `Experimental CommonJS graph root ${JSON.stringify(rootPath)} is an ES module.` + ); + } + + const modules: ExperimentalCommonJsGraphModule[] = []; + const seen = new Set(); + const collect = (sourcePath: string) => { + if (seen.has(sourcePath)) { + return; + } + seen.add(sourcePath); + const record = this.#records.get(sourcePath) as ModuleRecord; + modules.push(record.module as ExperimentalCommonJsGraphModule); + for (const dependency of record.dependencies) { + collect(dependency); + } + }; + collect(rootPath); + + for (const module of modules) { + module.namedExports = [ + ...this.#collectNamedExports(module.sourcePath, new Set()), + ].sort(); + } + modules.sort((a, b) => a.emittedName.localeCompare(b.emittedName)); + return { root, modules }; + } + + async #visit( + sourcePath: string, + ancestors: Set + ): Promise { + const existing = this.#records.get(sourcePath); + if (existing !== undefined) { + if (!ancestors.has(sourcePath)) { + await existing.complete; + } + return existing.module as ExperimentalCommonJsGraphModule; + } + + const { + promise: complete, + resolve: resolveComplete, + reject: rejectComplete, + } = Promise.withResolvers(); + void complete.catch(() => {}); + const record: ModuleRecord = { + dependencies: [], + dependenciesBySpecifier: new Map(), + reexports: [], + directNamedExports: [], + complete, + }; + this.#records.set(sourcePath, record); + + try { + await this.#buildRecord(record, sourcePath, ancestors); + resolveComplete(); + } catch (error) { + this.#records.delete(sourcePath); + rejectComplete(error); + throw error; + } + return record.module as ExperimentalCommonJsGraphModule; + } + + async #buildRecord( + record: ModuleRecord, + sourcePath: string, + ancestors: Set + ): Promise { + const packageInfo = await findPackageInfo(sourcePath); + const emittedName = emittedNameFor(sourcePath, packageInfo); + const extension = path.extname(sourcePath); + const originalSource = await readFile(sourcePath, "utf8"); + + if (extension === ".json") { + JSON.parse(originalSource); + record.module = { + sourcePath, + emittedName, + transformedSource: `module.exports = JSON.parse(${JSON.stringify(originalSource)});\n`, + sourceType: "commonjs", + namedExports: [], + }; + return; + } + + const sourceType = await experimental_classifyJavaScriptFile(sourcePath); + record.module = { + sourcePath, + emittedName, + transformedSource: originalSource, + sourceType, + namedExports: [], + }; + if (sourceType === "esmodule") { + return; + } + + const calls = findRequireCalls(sourcePath, originalSource); + const namedExports = await parseNamedExports(originalSource); + record.directNamedExports = namedExports.exports; + record.reexports = namedExports.reexports; + const nextAncestors = new Set(ancestors).add(sourcePath); + const replacements: Array<{ + argument: AstNode; + emittedSpecifier: string; + }> = []; + + for (const call of calls) { + if (isBuiltin(call.specifier)) { + continue; + } + const resolved = await this.#resolve(call.specifier, sourcePath); + if (resolved === undefined) { + throw new Error( + `Experimental CommonJS graph could not resolve ${JSON.stringify(call.specifier)} from ${JSON.stringify(sourcePath)}.` + ); + } + const dependencyPath = path.resolve(resolved); + const dependency = await this.#visit(dependencyPath, nextAncestors); + if (dependency.sourceType === "esmodule") { + throw new Error( + `Experimental CommonJS graph cannot preserve require(${JSON.stringify(call.specifier)}) in ${JSON.stringify(sourcePath)} because it resolves to ES module ${JSON.stringify(dependencyPath)}.` + ); + } + record.dependencies.push(dependencyPath); + record.dependenciesBySpecifier.set(call.specifier, dependencyPath); + const relativeName = path.posix.relative( + path.posix.dirname(emittedName), + dependency.emittedName + ); + replacements.push({ + argument: call.argument, + emittedSpecifier: relativeName.startsWith(".") + ? relativeName + : `./${relativeName}`, + }); + } + record.module.transformedSource = rewriteRequires( + originalSource, + replacements + ); + } + + #collectNamedExports(sourcePath: string, seen: Set): Set { + if (seen.has(sourcePath)) { + return new Set(); + } + seen.add(sourcePath); + const record = this.#records.get(sourcePath) as ModuleRecord; + const names = new Set(record.directNamedExports); + for (const reexport of record.reexports) { + const dependencyPath = record.dependenciesBySpecifier.get(reexport); + if (dependencyPath !== undefined) { + for (const name of this.#collectNamedExports(dependencyPath, seen)) { + names.add(name); + } + } + } + return names; + } +} + +export function experimental_createCommonJsGraph( + options: ExperimentalCommonJsGraphOptions +): ExperimentalCommonJsGraphBuilder { + return new ExperimentalCommonJsGraphBuilder(options); +} diff --git a/packages/workers-utils/src/index.ts b/packages/workers-utils/src/index.ts index abdaf480490..d47d381dd32 100644 --- a/packages/workers-utils/src/index.ts +++ b/packages/workers-utils/src/index.ts @@ -176,3 +176,16 @@ export { _forceColour, formatZodError } from "./zod-format"; export { toUrlPath } from "./url-path"; export type { UrlPath } from "./url-path"; + +export { + ExperimentalCommonJsGraphBuilder, + experimental_classifyJavaScriptFile, + experimental_createCommonJsGraph, +} from "./commonjs-module-graph"; +export type { + ExperimentalCommonJsGraph, + ExperimentalCommonJsGraphModule, + ExperimentalCommonJsGraphOptions, + ExperimentalCommonJsResolver, + ExperimentalJavaScriptSourceType, +} from "./commonjs-module-graph"; diff --git a/packages/workers-utils/tests/commonjs-module-graph.test.ts b/packages/workers-utils/tests/commonjs-module-graph.test.ts new file mode 100644 index 00000000000..6b80e5a1fb6 --- /dev/null +++ b/packages/workers-utils/tests/commonjs-module-graph.test.ts @@ -0,0 +1,339 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, it, vi } from "vitest"; +import { + experimental_classifyJavaScriptFile, + experimental_createCommonJsGraph, +} from "../src/commonjs-module-graph"; +import { removeDirSync } from "../src/fs-helpers"; + +describe("experimental CommonJS module graph", () => { + let tempDirectory: string; + + beforeEach(() => { + tempDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), "workers-cjs-graph-") + ); + }); + + afterEach(() => { + removeDirSync(tempDirectory); + }); + + function write(relativePath: string, contents: string): string { + const filePath = path.join(tempDirectory, relativePath); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, contents); + return filePath; + } + + function writePackage( + relativeDirectory: string, + name: string, + type?: "commonjs" | "module" + ): string { + const packageDirectory = path.join(tempDirectory, relativeDirectory); + write( + path.join(relativeDirectory, "package.json"), + JSON.stringify({ name, ...(type === undefined ? {} : { type }) }) + ); + return packageDirectory; + } + + it("classifies extensions and the nearest package type", async ({ + expect, + }) => { + const packageDirectory = writePackage("node_modules/pkg", "pkg", "module"); + const cjs = write("node_modules/pkg/index.cjs", ""); + const mjs = write("node_modules/pkg/index.mjs", ""); + const esmJs = write("node_modules/pkg/index.js", ""); + write( + "node_modules/pkg/lib/package.json", + JSON.stringify({ type: "commonjs" }) + ); + const cjsJs = write("node_modules/pkg/lib/nested/index.js", ""); + + expect(packageDirectory).toBe(path.dirname(esmJs)); + await expect(experimental_classifyJavaScriptFile(cjs)).resolves.toBe( + "commonjs" + ); + await expect(experimental_classifyJavaScriptFile(mjs)).resolves.toBe( + "esmodule" + ); + await expect(experimental_classifyJavaScriptFile(esmJs)).resolves.toBe( + "esmodule" + ); + await expect(experimental_classifyJavaScriptFile(cjsJs)).resolves.toBe( + "commonjs" + ); + }); + + it("preserves package directories and rewrites relative requires", async ({ + expect, + }) => { + writePackage("node_modules/pkg", "pkg"); + const root = write( + "node_modules/pkg/lib/entry.cjs", + `module.exports = require("../shared/value.cjs");` + ); + const dependency = write( + "node_modules/pkg/shared/value.cjs", + "module.exports = 42;" + ); + const graph = await experimental_createCommonJsGraph({ + resolve: async (specifier, importer) => + path.resolve(path.dirname(importer), specifier), + }).discover(root); + const dependencyModule = graph.modules.find( + (module) => module.sourcePath === dependency + ); + + expect(graph.modules).toHaveLength(2); + expect(graph.root.emittedName).toMatch( + /^__cloudflare_cjs__\/[a-f0-9]{10}\/pkg\/lib\/entry\.cjs$/ + ); + expect(dependencyModule?.emittedName).toMatch( + /^__cloudflare_cjs__\/[a-f0-9]{10}\/pkg\/shared\/value\.cjs$/ + ); + expect(graph.root.transformedSource).toBe( + `module.exports = require("../shared/value.cjs");` + ); + }); + + it("delegates extensionless resolution to the caller", async ({ expect }) => { + writePackage("node_modules/pkg", "pkg"); + const root = write( + "node_modules/pkg/index.cjs", + `module.exports = require("./value");` + ); + const dependency = write( + "node_modules/pkg/value.js", + "module.exports = 1;" + ); + const resolve = vi.fn(async () => dependency); + const graph = await experimental_createCommonJsGraph({ resolve }).discover( + root + ); + + expect(resolve).toHaveBeenCalledWith("./value", root); + expect(graph.root.transformedSource).toContain('require("./value.js")'); + }); + + it("converts required JSON to CommonJS source", async ({ expect }) => { + writePackage("node_modules/pkg", "pkg"); + const root = write( + "node_modules/pkg/index.cjs", + `module.exports = require("./data.json");` + ); + const jsonSource = '{"answer":42,"__proto__":{"safe":true}}'; + const json = write("node_modules/pkg/data.json", jsonSource); + const graph = await experimental_createCommonJsGraph({ + resolve: async () => json, + }).discover(root); + const jsonModule = graph.modules.find( + (module) => module.sourcePath === json + ); + + expect(jsonModule).toMatchObject({ sourceType: "commonjs" }); + expect(jsonModule?.transformedSource).toBe( + `module.exports = JSON.parse(${JSON.stringify(jsonSource)});\n` + ); + }); + + it("leaves Node builtins and non-global require calls untouched", async ({ + expect, + }) => { + writePackage("node_modules/pkg", "pkg"); + const source = [ + `const fs = require("node:fs");`, + `const path = require("path");`, + `object.require("./not-a-dependency");`, + `function useLocal(require, name) { return require(name); }`, + ].join("\n"); + const root = write("node_modules/pkg/index.cjs", source); + const resolve = vi.fn(); + const graph = await experimental_createCommonJsGraph({ resolve }).discover( + root + ); + + expect(resolve).not.toHaveBeenCalled(); + expect(graph.root.transformedSource).toBe(source); + }); + + it("supports optional catch bindings", async ({ expect }) => { + writePackage("node_modules/pkg", "pkg"); + const root = write( + "node_modules/pkg/index.cjs", + `try { require("./child.cjs"); } catch {}` + ); + const child = write( + "node_modules/pkg/child.cjs", + `module.exports = "child";` + ); + const graph = await experimental_createCommonJsGraph({ + resolve: async () => child, + }).discover(root); + + expect(graph.modules).toHaveLength(2); + }); + + it("preserves cycles while rewriting both edges", async ({ expect }) => { + writePackage("node_modules/pkg", "pkg"); + const a = write( + "node_modules/pkg/a.cjs", + `exports.a = require("./b.cjs");` + ); + const b = write( + "node_modules/pkg/b.cjs", + `exports.b = require("./a.cjs");` + ); + const graph = await experimental_createCommonJsGraph({ + resolve: async (specifier, importer) => + path.resolve(path.dirname(importer), specifier), + }).discover(a); + const aModule = graph.modules.find((module) => module.sourcePath === a); + const bModule = graph.modules.find((module) => module.sourcePath === b); + + expect(graph.modules).toHaveLength(2); + expect(aModule?.transformedSource).toContain('require("./b.cjs")'); + expect(bModule?.transformedSource).toContain('require("./a.cjs")'); + }); + + it("serializes concurrent discovery of cyclic roots", async ({ expect }) => { + writePackage("node_modules/pkg", "pkg"); + const a = write( + "node_modules/pkg/a.cjs", + `exports.a = require("./b.cjs");` + ); + const b = write( + "node_modules/pkg/b.cjs", + `exports.b = require("./a.cjs");` + ); + const builder = experimental_createCommonJsGraph({ + resolve: async (specifier, importer) => + path.resolve(path.dirname(importer), specifier), + }); + + const graphs = await Promise.all([ + builder.discover(a), + builder.discover(b), + ]); + + expect(graphs.map((graph) => graph.modules.length)).toEqual([2, 2]); + }); + + it("rejects CommonJS requires that resolve to an ES module", async ({ + expect, + }) => { + writePackage("node_modules/pkg", "pkg"); + const root = write( + "node_modules/pkg/index.cjs", + `module.exports = require("./dependency.mjs");` + ); + const dependency = write( + "node_modules/pkg/dependency.mjs", + `export default 42;` + ); + + await expect( + experimental_createCommonJsGraph({ + resolve: async () => dependency, + }).discover(root) + ).rejects.toThrow(/resolves to ES module/); + }); + + it("returns only safe lexer-derived named exports, including reexports", async ({ + expect, + }) => { + writePackage("node_modules/pkg", "pkg"); + const root = write( + "node_modules/pkg/index.cjs", + `module.exports = require("./exports.cjs");` + ); + const dependency = write( + "node_modules/pkg/exports.cjs", + [ + `exports.good = true;`, + `exports["alsoGood"] = true;`, + `exports["not-valid"] = true;`, + `exports.default = true;`, + `exports.__esModule = true;`, + `exports.class = true;`, + `exports["injected = 0; export const unsafe"] = true;`, + ].join("\n") + ); + const graph = await experimental_createCommonJsGraph({ + resolve: async () => dependency, + }).discover(root); + + expect(graph.root.namedExports).toEqual(["alsoGood", "good"]); + }); + + it("assigns distinct stable names to duplicate package instances", async ({ + expect, + }) => { + writePackage("node_modules/a/node_modules/duplicate", "duplicate"); + writePackage("node_modules/b/node_modules/duplicate", "duplicate"); + const first = write( + "node_modules/a/node_modules/duplicate/index.cjs", + "module.exports = 1;" + ); + const second = write( + "node_modules/b/node_modules/duplicate/index.cjs", + "module.exports = 2;" + ); + const builder = experimental_createCommonJsGraph({ + resolve: async () => undefined, + }); + const firstGraph = await builder.discover(first); + const secondGraph = await builder.discover(second); + + expect(firstGraph.root.emittedName).not.toBe(secondGraph.root.emittedName); + expect(firstGraph.root.emittedName).toMatch( + /^__cloudflare_cjs__\/[a-f0-9]{10}\/duplicate\/index\.cjs$/ + ); + expect(secondGraph.root.emittedName).toMatch( + /^__cloudflare_cjs__\/[a-f0-9]{10}\/duplicate\/index\.cjs$/ + ); + }); + + it("caches shared modules across multiple roots", async ({ expect }) => { + writePackage("node_modules/pkg", "pkg"); + const first = write( + "node_modules/pkg/first.cjs", + `module.exports = require("./shared.cjs");` + ); + const second = write( + "node_modules/pkg/second.cjs", + `module.exports = require("./shared.cjs");` + ); + const shared = write( + "node_modules/pkg/shared.cjs", + "module.exports = true;" + ); + const resolve = vi.fn(async () => shared); + const builder = experimental_createCommonJsGraph({ resolve }); + + await builder.discover(first); + const graph = await builder.discover(second); + + expect(resolve).toHaveBeenCalledTimes(2); + expect(graph.modules).toHaveLength(2); + }); + + it("throws a clear diagnostic for dynamic require", async ({ expect }) => { + writePackage("node_modules/pkg", "pkg"); + const root = write( + "node_modules/pkg/index.cjs", + "const name = './value.cjs';\nmodule.exports = require(name);" + ); + const graph = experimental_createCommonJsGraph({ + resolve: async () => undefined, + }); + + await expect(graph.discover(root)).rejects.toThrow( + /Experimental CommonJS graph cannot analyze dynamic require\(\).*index\.cjs:2:18.*one string literal/ + ); + }); +}); diff --git a/packages/wrangler/src/__tests__/create-worker-upload-form/metadata.test.ts b/packages/wrangler/src/__tests__/create-worker-upload-form/metadata.test.ts index ce85d6a509a..4a7c711e62b 100644 --- a/packages/wrangler/src/__tests__/create-worker-upload-form/metadata.test.ts +++ b/packages/wrangler/src/__tests__/create-worker-upload-form/metadata.test.ts @@ -83,6 +83,27 @@ describe("createWorkerUploadForm — basic structure", () => { expect(utilsPart.type).toBe("application/javascript+module"); }); + it("should upload additional CommonJS modules as JavaScript parts", ({ + expect, + }) => { + const form = createWorkerUploadForm( + createEsmWorker({ + modules: [ + { + name: "dependency.js", + filePath: "dependency.js", + content: "module.exports = {};", + type: "commonjs", + }, + ], + }), + {} + ); + const dependencyPart = form.get("dependency.js") as File; + expect(dependencyPart).not.toBeNull(); + expect(dependencyPart.type).toBe("application/javascript"); + }); + it("should throw when commonjs worker has additional modules", ({ expect, }) => { diff --git a/packages/wrangler/src/__tests__/module-collection.test.ts b/packages/wrangler/src/__tests__/module-collection.test.ts new file mode 100644 index 00000000000..035600d20cd --- /dev/null +++ b/packages/wrangler/src/__tests__/module-collection.test.ts @@ -0,0 +1,235 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { runInTempDir, seed } from "@cloudflare/workers-utils/test-helpers"; +import { describe, it } from "vitest"; +import { bundleWorker, type BundleOptions } from "../deployment-bundle/bundle"; +import { createModuleCollector } from "../deployment-bundle/module-collection"; +import type { Entry } from "@cloudflare/workers-utils"; + +const moduleEntrySource = ` +import fixtureDefault from "fixture"; +import { fixtureNamed } from "./named.js"; +const requiredFixture = require("fixture"); +export default { + fetch() { + return Response.json({ + value: fixtureDefault(), + named: fixtureNamed, + required: requiredFixture.named, + }); + } +}; +`; + +const serviceWorkerEntrySource = ` +const fixture = require("fixture"); +addEventListener("fetch", (event) => event.respondWith(new Response(fixture()))); +`; + +async function seedCommonJsPackage( + entrySource: string, + includeNodeBuiltin = true +): Promise { + await seed({ + "src/index.js": entrySource, + "src/named.js": ` + import { named } from "fixture"; + export const fixtureNamed = named; + `, + "node_modules/fixture/package.json": JSON.stringify({ + name: "fixture", + main: "index.js", + }), + "node_modules/fixture/index.js": ` + const path = ${includeNodeBuiltin ? 'require("path")' : '{ sep: "/" }'}; + const child = require("./child.js"); + const cycle = require("./cycle-a.js"); + function fixtureDefault() { + return ["CJS_ROOT", path.sep, child, cycle.value].join(":"); + } + module.exports = fixtureDefault; + module.exports.named = "named-export"; + `, + "node_modules/fixture/child.js": ` + const data = require("./data.json"); + module.exports = "CJS_CHILD:" + data.value; + `, + "node_modules/fixture/data.json": JSON.stringify({ + marker: "CJS_JSON", + value: "json-value", + }), + "node_modules/fixture/cycle-a.js": ` + exports.value = "CJS_CYCLE_A"; + exports.other = require("./cycle-b.js"); + `, + "node_modules/fixture/cycle-b.js": ` + exports.value = "CJS_CYCLE_B"; + exports.other = require("./cycle-a.js"); + `, + }); +} + +async function bundleFixture({ + compatibilityFlags, + format = "modules", + bundle = true, + destination = "dist", +}: { + compatibilityFlags: string[]; + format?: Entry["format"]; + bundle?: boolean; + destination?: string; +}) { + const entry: Entry = { + file: path.resolve("src/index.js"), + projectRoot: process.cwd(), + configPath: undefined, + format, + moduleRoot: path.resolve("src"), + exports: [], + }; + const moduleCollector = createModuleCollector({ + entry, + findAdditionalModules: false, + }); + const options: BundleOptions = { + bundle, + additionalModules: [], + moduleCollector, + doBindings: [], + workflowBindings: [], + jsxFactory: undefined, + jsxFragment: undefined, + entryName: undefined, + watch: undefined, + tsconfig: undefined, + minify: false, + keepNames: true, + nodejsCompatMode: "v2", + compatibilityDate: "2026-08-02", + compatibilityFlags, + define: {}, + alias: {}, + checkFetch: false, + targetConsumer: "deploy", + testScheduled: undefined, + inject: undefined, + sourcemap: false, + plugins: undefined, + isOutfile: undefined, + local: false, + projectRoot: process.cwd(), + defineNavigatorUserAgent: false, + external: undefined, + metafile: undefined, + }; + + return bundleWorker(entry, path.resolve(destination), options); +} + +describe("experimental CommonJS module collection", () => { + runInTempDir(); + + it("keeps bundling CommonJS dependencies when the flag is disabled", async ({ + expect, + }) => { + await seedCommonJsPackage(moduleEntrySource); + const result = await bundleFixture({ + compatibilityFlags: ["nodejs_compat"], + }); + const entrySource = await readFile(result.resolvedEntryPointPath, "utf8"); + + expect(result.modules).toEqual([]); + expect(entrySource).toContain("CJS_ROOT"); + expect(entrySource).toContain("CJS_CHILD"); + expect(entrySource).toContain("CJS_JSON"); + }); + + it("externalizes a complete CommonJS graph behind an ESM interop wrapper", async ({ + expect, + }) => { + await seedCommonJsPackage(moduleEntrySource); + const result = await bundleFixture({ + compatibilityFlags: ["nodejs_compat", "new_module_registry"], + }); + const entrySource = await readFile(result.resolvedEntryPointPath, "utf8"); + const writtenModules = await Promise.all( + result.modules.map((module) => + readFile( + path.resolve( + path.dirname(result.resolvedEntryPointPath), + module.name + ), + "utf8" + ) + ) + ); + const root = result.modules.find((module) => + module.content.toString().includes("CJS_ROOT") + ); + const json = result.modules.find((module) => + module.content.toString().includes("CJS_JSON") + ); + const cycleA = result.modules.find((module) => + module.content.toString().includes("CJS_CYCLE_A") + ); + const cycleB = result.modules.find((module) => + module.content.toString().includes("CJS_CYCLE_B") + ); + + expect(result.modules).toHaveLength(5); + expect(result.modules.every((module) => module.type === "commonjs")).toBe( + true + ); + expect(writtenModules).toEqual( + result.modules.map((module) => module.content.toString()) + ); + expect(new Set(result.modules.map((module) => module.name)).size).toBe(5); + expect(root?.name).toMatch( + /^__cloudflare_cjs__\/[a-f0-9]{10}\/fixture\/index\.js$/ + ); + expect(root?.content).toContain('require("path")'); + expect(root?.content).toContain('require("./child.js")'); + expect(json?.content).toMatch(/^module\.exports = JSON\.parse\(/); + expect(cycleA?.content).toContain('require("./cycle-b.js")'); + expect(cycleB?.content).toContain('require("./cycle-a.js")'); + + expect(entrySource).not.toContain("CJS_ROOT"); + expect(entrySource).not.toContain("CJS_CHILD"); + expect(entrySource).not.toContain("CJS_JSON"); + expect(entrySource).toContain( + `import __commonJsModule from ${JSON.stringify(`./${root?.name}`)}` + ); + expect(entrySource).toContain("var fixture_default = __commonJsModule"); + expect(entrySource).toContain("value: fixture_default()"); + expect(entrySource).toMatch(/var named = __commonJsModule\.named/); + expect(entrySource).toMatch(/module\.exports = __commonJsModule\d*;/); + }); + + it.for([ + { + label: "service-worker format", + format: "service-worker" as const, + bundle: true, + entrySource: serviceWorkerEntrySource, + }, + { + label: "no-bundle builds", + format: "modules" as const, + bundle: false, + entrySource: moduleEntrySource, + }, + ])("does not activate for $label", async (testCase, { expect }) => { + await seedCommonJsPackage( + testCase.entrySource, + testCase.format === "modules" + ); + const result = await bundleFixture({ + compatibilityFlags: ["nodejs_compat", "new_module_registry"], + format: testCase.format, + bundle: testCase.bundle, + }); + + expect(result.modules).toEqual([]); + }); +}); diff --git a/packages/wrangler/src/deployment-bundle/bundle.ts b/packages/wrangler/src/deployment-bundle/bundle.ts index ee99209fa73..857ba18a7f5 100644 --- a/packages/wrangler/src/deployment-bundle/bundle.ts +++ b/packages/wrangler/src/deployment-bundle/bundle.ts @@ -23,7 +23,10 @@ import { cloudflareInternalPlugin } from "./esbuild-plugins/cloudflare-internal" import { configProviderPlugin } from "./esbuild-plugins/config-provider"; import { getNodeJSCompatPlugins } from "./esbuild-plugins/nodejs-plugins"; import { writeAdditionalModules } from "./find-additional-modules"; -import { noopModuleCollector } from "./module-collection"; +import { + createExperimentalCommonJsModulePlugin, + noopModuleCollector, +} from "./module-collection"; import type { MiddlewareLoader } from "./apply-middleware"; import type { ModuleCollector } from "./module-collection"; import type { @@ -412,6 +415,11 @@ export async function bundleWorker( plugins: [ aliasPlugin, moduleCollector.plugin, + ...(bundle && + entry.format === "modules" && + compatibilityFlags?.includes("new_module_registry") + ? [createExperimentalCommonJsModulePlugin(moduleCollector.modules)] + : []), ...getNodeJSCompatPlugins({ mode: nodejsCompatMode ?? null, compatibilityDate, diff --git a/packages/wrangler/src/deployment-bundle/module-collection.ts b/packages/wrangler/src/deployment-bundle/module-collection.ts index bb796b33044..1ef57f1ab9a 100644 --- a/packages/wrangler/src/deployment-bundle/module-collection.ts +++ b/packages/wrangler/src/deployment-bundle/module-collection.ts @@ -3,7 +3,11 @@ import crypto from "node:crypto"; import { readdirSync } from "node:fs"; import { readFile } from "node:fs/promises"; import path from "node:path"; -import { UserError } from "@cloudflare/workers-utils"; +import { + experimental_classifyJavaScriptFile, + experimental_createCommonJsGraph, + UserError, +} from "@cloudflare/workers-utils"; import globToRegExp from "glob-to-regexp"; import { sync as resolveSync } from "resolve"; import { logger } from "../logger"; @@ -18,6 +22,7 @@ import type { CfModuleType, Config, ConfigModuleRuleType, + ExperimentalCommonJsGraph, } from "@cloudflare/workers-utils"; import type esbuild from "esbuild"; @@ -84,6 +89,200 @@ export const noopModuleCollector: ModuleCollector = { }, }; +const commonJsInteropNamespace = "wrangler-commonjs-interop"; +const commonJsRootKinds = new Set([ + "import-statement", + "dynamic-import", + "require-call", +]); +const commonJsGraphPluginData = { + skip: true, + experimentalCommonJsGraph: true, +}; + +function isCommonJsGraphResolution(pluginData: unknown): boolean { + return ( + typeof pluginData === "object" && + pluginData !== null && + "experimentalCommonJsGraph" in pluginData + ); +} + +function isBareModuleSpecifier(specifier: string): boolean { + return ( + !specifier.startsWith(".") && + !specifier.startsWith("/") && + !specifier.startsWith("\\") && + !path.isAbsolute(specifier) + ); +} + +function isInNodeModules(filePath: string): boolean { + return filePath.split(path.sep).includes("node_modules"); +} + +function createCommonJsInteropWrapper( + graph: ExperimentalCommonJsGraph, + kind: "import" | "require" +): string { + let binding = "__commonJsModule"; + while (graph.root.namedExports.includes(binding)) { + binding = `_${binding}`; + } + + if (kind === "require") { + return [ + `import ${binding} from ${JSON.stringify(`./${graph.root.emittedName}`)};`, + `module.exports = ${binding};`, + ].join("\n"); + } + + return [ + `import ${binding} from ${JSON.stringify(`./${graph.root.emittedName}`)};`, + `export default ${binding};`, + ...graph.root.namedExports.map( + (name) => `export const ${name} = ${binding}.${name};` + ), + ].join("\n"); +} + +/** + * Preserves npm CommonJS dependency boundaries for workerd's experimental + * module registry. The caller is responsible for applying the feature gate. + */ +export function createExperimentalCommonJsModulePlugin( + modules: CfModule[] +): esbuild.Plugin { + return { + name: "wrangler-experimental-commonjs-module-collector", + setup(build) { + let emittedModuleNames = new Set(); + let roots = new Map>(); + let wrappers = new Map< + string, + { rootPath: string; kind: "import" | "require" } + >(); + let graphBuilder = experimental_createCommonJsGraph({ + resolve: resolveGraphEdge, + }); + + async function resolveGraphEdge( + specifier: string, + importer: string + ): Promise { + const result = await build.resolve(specifier, { + kind: "require-call", + importer, + resolveDir: path.dirname(importer), + pluginData: commonJsGraphPluginData, + }); + return result.errors.length === 0 && + !result.external && + result.namespace === "file" && + path.isAbsolute(result.path) + ? result.path + : undefined; + } + + async function discoverGraph( + rootPath: string + ): Promise { + let graphPromise = roots.get(rootPath); + if (graphPromise === undefined) { + graphPromise = graphBuilder.discover(rootPath); + roots.set(rootPath, graphPromise); + } + const graph = await graphPromise; + for (const module of graph.modules) { + if (emittedModuleNames.has(module.emittedName)) { + continue; + } + emittedModuleNames.add(module.emittedName); + modules.push({ + name: module.emittedName, + filePath: module.sourcePath, + content: module.transformedSource, + type: module.sourceType === "commonjs" ? "commonjs" : "esm", + }); + } + return graph; + } + + build.onStart(() => { + emittedModuleNames = new Set(); + roots = new Map(); + wrappers = new Map(); + graphBuilder = experimental_createCommonJsGraph({ + resolve: resolveGraphEdge, + }); + }); + + build.onResolve({ filter: /.*/ }, (args) => { + if ( + args.namespace === commonJsInteropNamespace && + args.path.startsWith("./__cloudflare_cjs__/") + ) { + return { path: args.path, external: true }; + } + }); + + build.onResolve({ filter: /.*/ }, async (args) => { + if ( + isCommonJsGraphResolution(args.pluginData) || + !commonJsRootKinds.has(args.kind) + ) { + return; + } + + const resolved = await build.resolve(args.path, { + kind: args.kind, + importer: args.importer, + resolveDir: args.resolveDir, + pluginData: commonJsGraphPluginData, + }); + if ( + resolved.errors.length > 0 || + resolved.external || + resolved.namespace !== "file" || + !path.isAbsolute(resolved.path) || + ![".js", ".cjs"].includes(path.extname(resolved.path)) || + (!isBareModuleSpecifier(args.path) && + !isInNodeModules(args.importer) && + !isInNodeModules(resolved.path)) || + (await experimental_classifyJavaScriptFile(resolved.path)) !== + "commonjs" + ) { + return; + } + + await discoverGraph(resolved.path); + const kind = args.kind === "require-call" ? "require" : "import"; + const wrapperPath = `${resolved.path}?wrangler-commonjs-${kind}`; + wrappers.set(wrapperPath, { rootPath: resolved.path, kind }); + return { + path: wrapperPath, + namespace: commonJsInteropNamespace, + sideEffects: resolved.sideEffects, + }; + }); + + build.onLoad( + { filter: /.*/, namespace: commonJsInteropNamespace }, + async (args) => { + const wrapper = wrappers.get(args.path); + assert(wrapper, `Missing CommonJS wrapper for ${args.path}`); + const graph = await discoverGraph(wrapper.rootPath); + return { + contents: createCommonJsInteropWrapper(graph, wrapper.kind), + loader: "js", + watchFiles: graph.modules.map((module) => module.sourcePath), + }; + } + ); + }, + }; +} + export function createModuleCollector(props: { entry: Entry; findAdditionalModules: boolean; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 833d296f9f7..84128467f28 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4342,6 +4342,12 @@ importers: packages/workers-utils: dependencies: + acorn: + specifier: 8.16.0 + version: 8.16.0 + cjs-module-lexer: + specifier: 1.2.3 + version: 1.2.3 undici: specifier: catalog:default version: 7.28.0