From 03606d6c8ca0a2771099053f0b7933a1e4f109c2 Mon Sep 17 00:00:00 2001 From: killa Date: Sun, 10 May 2026 10:30:47 +0800 Subject: [PATCH 1/4] feat(core): add manifest-backed loader fs --- packages/core/package.json | 1 + packages/core/src/loader/egg_loader.ts | 32 +- packages/core/src/loader/loader_fs.ts | 313 ++++++++++++++++++ .../test/__snapshots__/index.test.ts.snap | 1 + .../test/loader/manifest_coverage.test.ts | 4 + .../test/loader/manifest_loader_fs.test.ts | 164 +++++++++ 6 files changed, 509 insertions(+), 6 deletions(-) create mode 100644 packages/core/test/loader/manifest_loader_fs.test.ts diff --git a/packages/core/package.json b/packages/core/package.json index 5cb1429b6a..78b4a78271 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -49,6 +49,7 @@ "get-ready": "catalog:", "globby": "catalog:", "is-type-of": "catalog:", + "multimatch": "catalog:", "node-homedir": "catalog:", "performance-ms": "catalog:", "ready-callback": "catalog:", diff --git a/packages/core/src/loader/egg_loader.ts b/packages/core/src/loader/egg_loader.ts index 57b892ea55..fcc0dfb3b0 100644 --- a/packages/core/src/loader/egg_loader.ts +++ b/packages/core/src/loader/egg_loader.ts @@ -13,7 +13,7 @@ import { isAsyncFunction, isClass, isGeneratorFunction, isObject, isPromise } fr import { homedir } from 'node-homedir'; import { now, diff } from 'performance-ms'; import { register as tsconfigPathsRegister } from 'tsconfig-paths'; -import { getParamNames, readJSONSync, readJSON, exists } from 'utility'; +import { getParamNames, readJSONSync } from 'utility'; import type { BaseContextClass } from '../base_context_class.ts'; import type { Context, EggCore, MiddlewareFunc } from '../egg.ts'; @@ -24,7 +24,7 @@ import { sequencify } from '../utils/sequencify.ts'; import { Timing } from '../utils/timing.ts'; import { type ContextLoaderOptions, ContextLoader } from './context_loader.ts'; import { type FileLoaderOptions, CaseStyle, FULLPATH, FileLoader } from './file_loader.ts'; -import { RealLoaderFS, type LoaderFS } from './loader_fs.ts'; +import { ManifestLoaderFS, RealLoaderFS, type LoaderFS } from './loader_fs.ts'; import { ManifestStore, type StartupManifest } from './manifest.ts'; const debug = debuglog('egg/core/loader/egg_loader'); @@ -95,7 +95,10 @@ export class EggLoader { */ constructor(options: EggLoaderOptions) { this.options = options; - this.loaderFS = this.options.loaderFS ?? new RealLoaderFS(); + const bundleStore = ManifestStore.getBundleStore(); + this.loaderFS = + this.options.loaderFS ?? + (bundleStore?.baseDir === this.options.baseDir ? new ManifestLoaderFS(bundleStore) : new RealLoaderFS()); assert(fs.existsSync(this.options.baseDir), `${this.options.baseDir} not exists`); assert(this.options.app, 'options.app is required'); assert(this.options.logger, 'options.logger is required'); @@ -176,6 +179,9 @@ export class EggLoader { this.manifest = ManifestStore.load(this.options.baseDir, this.serverEnv, this.serverScope) ?? ManifestStore.createCollector(this.options.baseDir); + if (!this.options.loaderFS && !(this.loaderFS instanceof ManifestLoaderFS)) { + this.loaderFS = new ManifestLoaderFS(this.manifest, this.loaderFS); + } } get app(): EggCore { @@ -644,8 +650,8 @@ export class EggLoader { let pkg: any; let eggPluginConfig: any; const pluginPackage = path.join(plugin.path as string, 'package.json'); - if (await utils.existsPath(pluginPackage)) { - pkg = await readJSON(pluginPackage); + if (this.loaderFS.exists(pluginPackage)) { + pkg = this.loaderFS.readJSON(pluginPackage); eggPluginConfig = pkg.eggPlugin; if (pkg.version) { plugin.version = pkg.version; @@ -848,7 +854,7 @@ export class EggLoader { } else if (exports.require) { realPluginPath = path.join(pluginPath, exports.require); } - if (exports.typescript && isSupportTypeScript() && !(await exists(realPluginPath))) { + if (exports.typescript && isSupportTypeScript() && !this.loaderFS.exists(realPluginPath)) { // if require/import path not exists, use typescript path for development stage realPluginPath = path.join(pluginPath, exports.typescript); debug('[formatPluginPathFromPackageJSON] use typescript path %o', realPluginPath); @@ -1796,6 +1802,7 @@ export class EggLoader { */ #collectConventionalDynamicFiles(manifest: StartupManifest): void { for (const unit of this.getLoadUnits()) { + this.#collectConventionFile(manifest, path.join(unit.path, 'package.json')); for (const load of CONVENTIONAL_MANIFEST_LOADS) { const target = path.join(unit.path, ...load.path); if (load.type === 'resolve') { @@ -1838,6 +1845,19 @@ export class EggLoader { return manifest.fileDiscovery[dirKey]; } + #collectConventionFile(manifest: StartupManifest, filepath: string): void { + const fileKey = this.#toManifestRel(filepath); + if (Object.values(manifest.resolveCache).includes(fileKey)) return; + if (!fs.existsSync(filepath) || !fs.statSync(filepath).isFile()) return; + + const dirKey = this.#toManifestRel(path.dirname(filepath)); + const basename = path.basename(filepath); + const files = manifest.fileDiscovery[dirKey] ?? []; + if (!files.includes(basename)) { + manifest.fileDiscovery[dirKey] = [...files, basename].sort(); + } + } + #toManifestRel(filepath: string): string { const rel = path.isAbsolute(filepath) ? path.relative(this.options.baseDir, filepath) : filepath; return rel.replaceAll(path.sep, '/'); diff --git a/packages/core/src/loader/loader_fs.ts b/packages/core/src/loader/loader_fs.ts index 8446c60c1a..f8e3739d97 100644 --- a/packages/core/src/loader/loader_fs.ts +++ b/packages/core/src/loader/loader_fs.ts @@ -1,9 +1,13 @@ import fs, { type Stats } from 'node:fs'; +import path from 'node:path'; +import type {} from '@eggjs/typings/global'; import globby from 'globby'; +import multimatch from 'multimatch'; import { readJSONSync } from 'utility'; import utils from '../utils/index.ts'; +import type { ManifestStore } from './manifest.ts'; export type LoaderFSGlobOptions = globby.GlobbyOptions; @@ -41,3 +45,312 @@ export class RealLoaderFS implements LoaderFS { return utils.loadFile(filepath); } } + +export class ManifestLoaderFS implements LoaderFS { + readonly #manifest: ManifestStore; + readonly #fallback: LoaderFS; + + constructor(manifest: ManifestStore, fallback: LoaderFS = new RealLoaderFS()) { + this.#manifest = manifest; + this.#fallback = fallback; + } + + exists(filepath: string): boolean { + return this.#hasManifestEntry(filepath) || this.#fallback.exists(filepath); + } + + stat(filepath: string): Stats { + const entry = this.#getManifestEntry(filepath); + if (entry) { + return createVirtualStats(entry.type); + } + return this.#fallback.stat(filepath); + } + + realpath(filepath: string): string { + const entry = this.#getManifestEntry(filepath); + if (entry) { + return this.#toAbsolute(entry.rel); + } + return this.#fallback.realpath(filepath); + } + + readJSON(filepath: string): T { + const entry = this.#getManifestEntry(filepath); + if (entry?.type === 'file') { + const bundled = this.#loadBundledModule(entry.rel); + if (bundled !== undefined) { + return unwrapDefaultExport(bundled) as T; + } + } + return this.#fallback.readJSON(filepath); + } + + glob(patterns: string | string[], options?: LoaderFSGlobOptions): string[] { + const cwd = options?.cwd === undefined ? process.cwd() : String(options.cwd); + const absoluteCwd = path.resolve(cwd); + const cwdRel = this.#toRelative(absoluteCwd); + const manifestFiles = this.#listManifestFilesUnder(cwdRel); + if (manifestFiles === undefined) { + return this.#fallback.glob(patterns, options); + } + + const matched = filterManifestGlob(manifestFiles, patterns, options); + if (options?.absolute) { + return matched.map((file) => path.join(absoluteCwd, file)); + } + return matched; + } + + async loadFile(filepath: string): Promise { + const entry = this.#getManifestEntry(filepath); + if (entry?.type === 'file') { + const bundled = this.#loadBundledModule(entry.rel); + if (bundled !== undefined) { + return unwrapDefaultExport(bundled); + } + } + return this.#fallback.loadFile(filepath); + } + + #hasManifestEntry(filepath: string): boolean { + return this.#getManifestEntry(filepath) !== undefined; + } + + #getManifestEntry(filepath: string): ManifestEntry | undefined { + const rel = this.#toRelative(filepath); + const resolved = this.#resolveManifestFile(rel); + if (resolved) { + return { type: 'file', rel: resolved }; + } + if (this.#isManifestDirectory(rel)) { + return { type: 'directory', rel }; + } + } + + #resolveManifestFile(rel: string): string | undefined { + const cache = this.#manifest.data.resolveCache; + if (Object.hasOwn(cache, rel)) { + return cache[rel] ?? undefined; + } + + if (this.#isManifestFile(rel)) { + return rel; + } + + return this.#resolveFromFileDiscovery(rel); + } + + #resolveFromFileDiscovery(rel: string): string | undefined { + let matchedDir: string | undefined; + for (const dir of Object.keys(this.#manifest.data.fileDiscovery)) { + if ((rel === dir || rel.startsWith(dir + '/')) && (!matchedDir || dir.length > matchedDir.length)) { + matchedDir = dir; + } + } + if (!matchedDir || rel === matchedDir) return; + + const request = rel.slice(matchedDir.length + 1); + for (const file of this.#manifest.data.fileDiscovery[matchedDir]) { + if (file === request) { + return path.posix.join(matchedDir, file); + } + + const ext = path.posix.extname(file); + if (!ext || ext === '.map') continue; + + const extensionlessFile = file.slice(0, -ext.length); + if (extensionlessFile === request || extensionlessFile === `${request}/index`) { + return path.posix.join(matchedDir, file); + } + } + } + + #isManifestFile(rel: string): boolean { + for (const [dir, files] of Object.entries(this.#manifest.data.fileDiscovery)) { + if (files.includes(path.posix.relative(dir, rel))) { + return true; + } + } + return Object.values(this.#manifest.data.resolveCache).includes(rel); + } + + #isManifestDirectory(rel: string): boolean { + if (rel === '') { + return this.#hasManifestData(); + } + if (Object.hasOwn(this.#manifest.data.fileDiscovery, rel)) { + return true; + } + + for (const [dir, files] of Object.entries(this.#manifest.data.fileDiscovery)) { + if (dir.startsWith(rel + '/')) { + return true; + } + for (const file of files) { + const fullRel = path.posix.join(dir, file); + if (path.posix.dirname(fullRel) === rel || fullRel.startsWith(rel + '/')) { + return true; + } + } + } + + for (const target of Object.values(this.#manifest.data.resolveCache)) { + if (target && (path.posix.dirname(target) === rel || target.startsWith(rel + '/'))) { + return true; + } + } + return false; + } + + #listManifestFilesUnder(cwdRel: string): string[] | undefined { + if (!this.#isManifestDirectory(cwdRel)) return; + + const files = new Set(); + for (const [dir, entries] of Object.entries(this.#manifest.data.fileDiscovery)) { + const prefix = dir === cwdRel ? '' : relativePrefix(cwdRel, dir); + if (prefix === undefined) continue; + for (const entry of entries) { + files.add(path.posix.join(prefix, entry)); + } + } + + for (const target of Object.values(this.#manifest.data.resolveCache)) { + if (!target) continue; + const prefix = relativePrefix(cwdRel, path.posix.dirname(target)); + if (prefix !== undefined) { + files.add(path.posix.join(prefix, path.posix.basename(target))); + } + } + return [...files].sort(); + } + + #loadBundledModule(rel: string): unknown { + const loader = globalThis.__EGG_BUNDLE_MODULE_LOADER__; + if (!loader) return undefined; + + for (const key of this.#bundleKeys(rel)) { + const loaded = loader(key); + if (loaded !== undefined) { + return loaded; + } + } + } + + #bundleKeys(rel: string): string[] { + const abs = this.#toAbsolute(rel); + return [...new Set([rel, normalizePath(abs)])]; + } + + #hasManifestData(): boolean { + return ( + Object.keys(this.#manifest.data.fileDiscovery).length > 0 || + Object.keys(this.#manifest.data.resolveCache).some((key) => this.#manifest.data.resolveCache[key] !== null) + ); + } + + #toRelative(filepath: string): string { + const rel = path.isAbsolute(filepath) ? path.relative(this.#manifest.baseDir, filepath) : filepath; + return normalizePath(rel); + } + + #toAbsolute(rel: string): string { + return path.isAbsolute(rel) ? rel : path.join(this.#manifest.baseDir, rel); + } +} + +interface ManifestEntry { + type: 'file' | 'directory'; + rel: string; +} + +function normalizePath(filepath: string): string { + return filepath.replaceAll(path.sep, '/'); +} + +function relativePrefix(cwdRel: string, dirRel: string): string | undefined { + if (cwdRel === '') return dirRel; + if (dirRel === cwdRel) return ''; + if (dirRel.startsWith(cwdRel + '/')) return dirRel.slice(cwdRel.length + 1); +} + +function filterManifestGlob(files: string[], patterns: string | string[], options?: LoaderFSGlobOptions): string[] { + const patternList = Array.isArray(patterns) ? patterns : [patterns]; + if (!patternList.some((pattern) => !pattern.startsWith('!'))) return []; + + const ignoreList = Array.isArray(options?.ignore) + ? options.ignore.map(String) + : options?.ignore + ? [String(options.ignore)] + : []; + const normalizedPatterns = patternList + .map(normalizeAlternationGroups) + .concat(ignoreList.map((pattern) => `!${normalizeAlternationGroups(pattern)}`)); + return multimatch(files, normalizedPatterns); +} + +function normalizeAlternationGroups(pattern: string): string { + let normalized = ''; + for (let index = 0; index < pattern.length; index++) { + const char = pattern[index]; + if (char !== '(' || isExtglobPrefix(pattern[index - 1])) { + normalized += char; + continue; + } + + const end = pattern.indexOf(')', index + 1); + if (end === -1) { + normalized += char; + continue; + } + + const group = pattern.slice(index + 1, end); + if (group.includes('|')) { + normalized += `{${group.replaceAll('|', ',')}}`; + index = end; + } else { + normalized += char; + } + } + return normalized; +} + +function isExtglobPrefix(char: string | undefined): boolean { + return char === '@' || char === '!' || char === '?' || char === '+' || char === '*'; +} + +function unwrapDefaultExport(value: unknown): unknown { + let unwrapped = value; + if (isRecord(unwrapped) && isRecord(unwrapped.default) && unwrapped.default.__esModule === true) { + unwrapped = unwrapped.default; + } + if (isRecord(unwrapped) && 'default' in unwrapped) { + return unwrapped.default; + } + return unwrapped; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object'; +} + +function createVirtualStats(type: 'file' | 'directory'): Stats { + const stat = Object.create(fs.Stats.prototype) as Stats; + const isFile = type === 'file'; + const timestamp = new Date(0); + Object.defineProperties(stat, { + size: { value: 0 }, + atimeMs: { value: 0 }, + mtimeMs: { value: 0 }, + ctimeMs: { value: 0 }, + birthtimeMs: { value: 0 }, + atime: { value: timestamp }, + mtime: { value: timestamp }, + ctime: { value: timestamp }, + birthtime: { value: timestamp }, + isFile: { value: () => isFile }, + isDirectory: { value: () => !isFile }, + isSymbolicLink: { value: () => false }, + }); + return stat; +} diff --git a/packages/core/test/__snapshots__/index.test.ts.snap b/packages/core/test/__snapshots__/index.test.ts.snap index fc1467a86e..02ac30dd77 100644 --- a/packages/core/test/__snapshots__/index.test.ts.snap +++ b/packages/core/test/__snapshots__/index.test.ts.snap @@ -18,6 +18,7 @@ exports[`should expose properties 1`] = ` "KoaRequest", "KoaResponse", "Lifecycle", + "ManifestLoaderFS", "ManifestStore", "RealLoaderFS", "Request", diff --git a/packages/core/test/loader/manifest_coverage.test.ts b/packages/core/test/loader/manifest_coverage.test.ts index 23a66288e6..4a7baf5d8e 100644 --- a/packages/core/test/loader/manifest_coverage.test.ts +++ b/packages/core/test/loader/manifest_coverage.test.ts @@ -175,6 +175,10 @@ describe('ManifestStore coverage: FileLoader getter auto-injects manifest', () = manifest.fileDiscovery['node_modules/@eggjs/security/app/middleware']?.includes('securities.js'), 'security middleware should be included in manifest fileDiscovery', ); + assert.ok( + manifest.fileDiscovery['node_modules/@eggjs/security']?.includes('package.json'), + 'plugin package metadata should be included in manifest fileDiscovery', + ); } finally { await testApp.close(); } diff --git a/packages/core/test/loader/manifest_loader_fs.test.ts b/packages/core/test/loader/manifest_loader_fs.test.ts new file mode 100644 index 0000000000..e833e242a4 --- /dev/null +++ b/packages/core/test/loader/manifest_loader_fs.test.ts @@ -0,0 +1,164 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { setBundleModuleLoader } from '@eggjs/utils'; +import { afterEach, describe, it } from 'vitest'; + +import { ManifestLoaderFS, RealLoaderFS, type LoaderFSGlobOptions } from '../../src/loader/loader_fs.ts'; +import { ManifestStore, type StartupManifest } from '../../src/loader/manifest.ts'; + +describe('test/loader/manifest_loader_fs.test.ts', () => { + const createdDirs: string[] = []; + + afterEach(async () => { + setBundleModuleLoader(undefined); + await Promise.all(createdDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); + }); + + it('serves manifest-backed stat, realpath, glob, readJSON, and loadFile from the bundle map', async () => { + const baseDir = await createTempDir(createdDirs, 'egg-manifest-loader-fs-'); + const manifest = createManifest({ + fileDiscovery: { + 'app/service': ['nested/order.ts', 'user.ts', 'user.d.ts'], + config: ['config.default.ts', 'plugin.ts'], + 'node_modules/fake-plugin': ['app.ts', 'package.json'], + }, + resolveCache: { + 'config/plugin': 'config/plugin.ts', + 'node_modules/fake-plugin/app': 'node_modules/fake-plugin/app.ts', + }, + }); + const store = ManifestStore.fromBundle(manifest, baseDir); + const modules: Record = { + 'config/plugin.ts': { default: { fakePlugin: { enable: true, package: 'fake-plugin' } } }, + [normalize(path.join(baseDir, 'config/plugin.ts'))]: { + default: { fakePlugin: { enable: true, package: 'fake-plugin' } }, + }, + 'node_modules/fake-plugin/package.json': { + default: { name: 'fake-plugin', version: '1.0.0', eggPlugin: { name: 'fakePlugin' } }, + }, + [normalize(path.join(baseDir, 'node_modules/fake-plugin/package.json'))]: { + default: { name: 'fake-plugin', version: '1.0.0', eggPlugin: { name: 'fakePlugin' } }, + }, + }; + setBundleModuleLoader((filepath) => modules[filepath]); + + const loaderFS = new ManifestLoaderFS(store); + const serviceFile = path.join(baseDir, 'app/service/user.ts'); + const serviceDir = path.join(baseDir, 'app/service'); + const pluginAlias = path.join(baseDir, 'config/plugin'); + const pluginPackage = path.join(baseDir, 'node_modules/fake-plugin/package.json'); + + assert.equal(loaderFS.exists(serviceFile), true); + assert.equal(loaderFS.exists(serviceDir), true); + assert.equal(loaderFS.exists(pluginAlias), true); + assert.equal(loaderFS.stat(serviceFile).isFile(), true); + const serviceDirStat = loaderFS.stat(serviceDir); + assert.equal(serviceDirStat.isDirectory(), true); + assert.equal(serviceDirStat.mtime.getTime(), 0); + assert.equal(loaderFS.realpath(pluginAlias), path.join(baseDir, 'config/plugin.ts')); + assert.deepEqual(loaderFS.glob(['**/*.(js|ts)', '!**/*.d.ts'], { cwd: serviceDir }), [ + 'nested/order.ts', + 'user.ts', + ]); + assert.deepEqual(loaderFS.glob('user.ts', { cwd: path.relative(process.cwd(), serviceDir), absolute: true }), [ + serviceFile, + ]); + assert.deepEqual(loaderFS.glob(['**/[no]*.ts', '!**/*.d.ts'], { cwd: serviceDir }), ['nested/order.ts']); + assert.deepEqual(loaderFS.readJSON(pluginPackage), { + name: 'fake-plugin', + version: '1.0.0', + eggPlugin: { name: 'fakePlugin' }, + }); + assert.deepEqual(await loaderFS.loadFile(pluginAlias), { fakePlugin: { enable: true, package: 'fake-plugin' } }); + }); + + it('falls back to the real filesystem for paths missing from the manifest', async () => { + const baseDir = await createTempDir(createdDirs, 'egg-manifest-loader-fs-fallback-'); + const realDir = path.join(baseDir, 'real'); + await fs.mkdir(realDir); + await fs.writeFile(path.join(realDir, 'package.json'), JSON.stringify({ name: 'real-package' })); + await fs.writeFile(path.join(realDir, 'config.js'), 'export default { real: true };\n'); + const store = ManifestStore.fromBundle(createManifest(), baseDir); + const loaderFS = new ManifestLoaderFS(store); + + assert.equal(loaderFS.exists(path.join(realDir, 'package.json')), true); + assert.equal(loaderFS.stat(path.join(realDir, 'package.json')).isFile(), true); + assert.deepEqual(loaderFS.glob('**/*.js', { cwd: realDir }), ['config.js']); + assert.deepEqual(loaderFS.readJSON(path.join(realDir, 'package.json')), { name: 'real-package' }); + assert.deepEqual(await loaderFS.loadFile(path.join(realDir, 'config.js')), { real: true }); + }); + + it('prefers manifest and bundle-map results over same-path real files', async () => { + const baseDir = await createTempDir(createdDirs, 'egg-manifest-loader-fs-priority-'); + await fs.mkdir(path.join(baseDir, 'config')); + await fs.writeFile(path.join(baseDir, 'config/plugin.js'), 'export default { source: "real" };\n'); + const manifest = createManifest({ + fileDiscovery: { + config: ['plugin.js'], + }, + }); + const store = ManifestStore.fromBundle(manifest, baseDir); + setBundleModuleLoader((filepath) => { + if (filepath === 'config/plugin.js' || filepath === normalize(path.join(baseDir, 'config/plugin.js'))) { + return { default: { source: 'manifest' } }; + } + }); + + const loaderFS = new ManifestLoaderFS(store); + + assert.deepEqual(await loaderFS.loadFile(path.join(baseDir, 'config/plugin.js')), { source: 'manifest' }); + }); + + it('does not fall back when manifest covers an empty directory', async () => { + const baseDir = await createTempDir(createdDirs, 'egg-manifest-loader-fs-empty-'); + const store = ManifestStore.fromBundle( + createManifest({ + fileDiscovery: { + empty: [], + }, + }), + baseDir, + ); + const loaderFS = new ManifestLoaderFS(store, new ThrowingGlobLoaderFS()); + + assert.deepEqual(loaderFS.glob('**/*.js', { cwd: path.join(baseDir, 'empty') }), []); + }); +}); + +class ThrowingGlobLoaderFS extends RealLoaderFS { + glob(_patterns: string | string[], _options?: LoaderFSGlobOptions): string[] { + throw new Error('unexpected real fs fallback'); + } +} + +async function createTempDir(createdDirs: string[], prefix: string): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + createdDirs.push(dir); + return dir; +} + +function createManifest( + overrides: Partial> = {}, +): StartupManifest { + return { + version: 1, + generatedAt: '2026-05-10T00:00:00.000Z', + invalidation: { + lockfileFingerprint: '', + configFingerprint: '', + serverEnv: 'unittest', + serverScope: '', + typescriptEnabled: true, + }, + extensions: {}, + resolveCache: overrides.resolveCache ?? {}, + fileDiscovery: overrides.fileDiscovery ?? {}, + }; +} + +function normalize(filepath: string): string { + return filepath.replaceAll(path.sep, '/'); +} From 165a360739c5319feff5ba764ca66a2bd1777703 Mon Sep 17 00:00:00 2001 From: killa Date: Sun, 10 May 2026 11:36:19 +0800 Subject: [PATCH 2/4] feat(bundler): inject manifest loader fs into worker --- packages/core/src/egg.ts | 4 +++ packages/core/test/egg.test.ts | 8 ++++- packages/egg/src/lib/start.ts | 4 ++- tools/egg-bundler/src/lib/EntryGenerator.ts | 8 +++-- tools/egg-bundler/test/EntryGenerator.test.ts | 33 ++++++++++++++++--- .../EntryGenerator.worker.canonical.snap | 8 +++-- 6 files changed, 52 insertions(+), 13 deletions(-) diff --git a/packages/core/src/egg.ts b/packages/core/src/egg.ts index 3cfccc0bcf..1d4c64c8b7 100644 --- a/packages/core/src/egg.ts +++ b/packages/core/src/egg.ts @@ -16,6 +16,7 @@ import type { ReadyFunctionArg } from 'get-ready'; import { BaseContextClass } from './base_context_class.ts'; import { Lifecycle } from './lifecycle.ts'; import { EggLoader } from './loader/egg_loader.ts'; +import type { LoaderFS } from './loader/loader_fs.ts'; import { Singleton, type SingletonCreateMethod, type SingletonOptions } from './singleton.ts'; import type { EggAppConfig } from './types.ts'; import utils, { type Fun } from './utils/index.ts'; @@ -33,6 +34,8 @@ export interface EggCoreOptions { env?: string; /** Skip lifecycle hooks, only trigger loadMetadata for manifest generation */ metadataOnly?: boolean; + /** Loader-facing filesystem abstraction */ + loaderFS?: LoaderFS; /** * When true, lifecycle stops after the `configWillLoad` phase. * `configDidLoad`, `didLoad`, `willReady`, `didReady`, and `serverDidReady` @@ -230,6 +233,7 @@ export class EggCore extends KoaApplication { env: options.env ?? '', EggCoreClass: EggCore, metadataOnly: options.metadataOnly, + loaderFS: options.loaderFS, }); } diff --git a/packages/core/test/egg.test.ts b/packages/core/test/egg.test.ts index c96cb9f068..eb3b3bacb8 100644 --- a/packages/core/test/egg.test.ts +++ b/packages/core/test/egg.test.ts @@ -9,7 +9,7 @@ import coffee from 'coffee'; import { mm } from 'mm'; import { describe, it, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; -import { EggCore } from '../src/index.js'; +import { EggCore, RealLoaderFS } from '../src/index.js'; import { createApp, getFilepath, type Application } from './helper.js'; describe('test/egg.test.ts', () => { @@ -54,6 +54,12 @@ describe('test/egg.test.ts', () => { assert.equal(app.loader.serverScope, 'scope'); }); + it('should pass options.loaderFS to EggLoader', () => { + const loaderFS = new RealLoaderFS(); + app = new EggCore({ loaderFS }); + assert.equal(app.loader.loaderFS, loaderFS); + }); + it('should not set value expect for application and agent', () => { assert.throws( () => diff --git a/packages/egg/src/lib/start.ts b/packages/egg/src/lib/start.ts index b17caf7288..32596c7233 100644 --- a/packages/egg/src/lib/start.ts +++ b/packages/egg/src/lib/start.ts @@ -1,6 +1,6 @@ import path from 'node:path'; -import { ManifestStore } from '@eggjs/core'; +import { ManifestStore, type LoaderFS } from '@eggjs/core'; import { importModule } from '@eggjs/utils'; import { readJSON } from 'utility'; @@ -20,6 +20,8 @@ export interface StartEggOptions { plugins?: EggPlugin; /** Skip lifecycle hooks, only trigger loadMetadata for manifest generation */ metadataOnly?: boolean; + /** Loader-facing filesystem abstraction */ + loaderFS?: LoaderFS; /** * When true, load application metadata for V8 startup snapshot construction. * The lifecycle stops after configWillLoad (no servers, timers, or connections) diff --git a/tools/egg-bundler/src/lib/EntryGenerator.ts b/tools/egg-bundler/src/lib/EntryGenerator.ts index b533f100ef..e24f0d8ef6 100644 --- a/tools/egg-bundler/src/lib/EntryGenerator.ts +++ b/tools/egg-bundler/src/lib/EntryGenerator.ts @@ -258,7 +258,7 @@ for (const [key, spec] of __EXTERNAL_SPECS) { /* eslint-disable */ import path from 'node:path'; -import { ManifestStore } from '@eggjs/core'; +import { ManifestLoaderFS, ManifestStore } from '@eggjs/core'; import type {} from '@eggjs/typings/global'; import { startEgg } from ${frameworkSpec}; import * as __frameworkModule from ${frameworkSpec}; @@ -315,12 +315,14 @@ for (const [appAbsRequest, targetRel] of __APP_RESOLVE_CACHE_ALIASES) { } } -ManifestStore.setBundleStore(ManifestStore.fromBundle(MANIFEST_DATA as any, __outputDir)); +const __bundleManifestStore = ManifestStore.fromBundle(MANIFEST_DATA as any, __outputDir); +const __loaderFS = new ManifestLoaderFS(__bundleManifestStore); +ManifestStore.setBundleStore(__bundleManifestStore); globalThis.__EGG_BUNDLE_MODULE_LOADER__ = (filepath) => { return __getBundleMap(filepath); }; -startEgg({ baseDir: __outputDir, framework: __framework, mode: 'single' }).then((app) => { +startEgg({ baseDir: __outputDir, framework: __framework, mode: 'single', loaderFS: __loaderFS }).then((app) => { const port = process.env.PORT || app.config.cluster?.listen?.port || 7001; app.listen(port, () => { // eslint-disable-next-line no-console diff --git a/tools/egg-bundler/test/EntryGenerator.test.ts b/tools/egg-bundler/test/EntryGenerator.test.ts index 6d3560c1d2..ebf00e34a7 100644 --- a/tools/egg-bundler/test/EntryGenerator.test.ts +++ b/tools/egg-bundler/test/EntryGenerator.test.ts @@ -224,13 +224,17 @@ describe('EntryGenerator', () => { const result = await gen.generate(); const worker = await fs.readFile(result.workerEntry, 'utf8'); - expect(worker).toContain("import { ManifestStore } from '@eggjs/core'"); + expect(worker).toContain("import { ManifestLoaderFS, ManifestStore } from '@eggjs/core'"); expect(worker).toContain('import { startEgg } from "egg"'); - expect(worker).toContain('ManifestStore.setBundleStore(ManifestStore.fromBundle(MANIFEST_DATA'); + expect(worker).toContain('const __bundleManifestStore = ManifestStore.fromBundle(MANIFEST_DATA'); + expect(worker).toContain('const __loaderFS = new ManifestLoaderFS(__bundleManifestStore)'); + expect(worker).toContain('ManifestStore.setBundleStore(__bundleManifestStore)'); expect(worker).toContain('__EGG_BUNDLE_MODULE_LOADER__'); expect(worker).toContain('__setBundleMap(__framework, __frameworkModule)'); expect(worker).not.toContain('__frameworkImport'); - expect(worker).toContain("startEgg({ baseDir: __outputDir, framework: __framework, mode: 'single' })"); + expect(worker).toContain( + "startEgg({ baseDir: __outputDir, framework: __framework, mode: 'single', loaderFS: __loaderFS })", + ); }); it('builds a BUNDLE_MAP keyed by relKey, output absolute, original app absolute, and resolveCache aliases', async () => { @@ -296,6 +300,11 @@ export const ManifestStore = { globalThis.__manifestStore = store; }, }; +export class ManifestLoaderFS { + constructor(store) { + this.store = store; + } +} `, ); await writePackage( @@ -315,6 +324,8 @@ export async function startEgg(options) { frameworkResolved: resolvedFramework?.frameworkMarker, frameworkStartEggMatches: resolvedFramework?.startEgg === startEgg, controllerResolved: resolvedController?.controllerMarker, + loaderFSBaseDir: options.loaderFS?.store?.baseDir, + loaderFSUsesManifestStore: options.loaderFS?.store === globalThis.__manifestStore, }, null, 2)); return { config: { cluster: { listen: { port: 0 } } }, @@ -349,21 +360,31 @@ export async function startEgg(options) { }); const runtimeResult = JSON.parse(await fs.readFile(resultFile, 'utf8')) as { - options: { baseDir: string; framework: string; mode: string }; + options: { baseDir: string; framework: string; mode: string; loaderFS: unknown }; manifestBaseDir: string; frameworkResolved: string; frameworkStartEggMatches: boolean; controllerResolved: string; + loaderFSBaseDir: string; + loaderFSUsesManifestStore: boolean; }; expect(runtimeResult.options).toEqual({ baseDir: outputDir, framework: '@runtime/framework', mode: 'single', + loaderFS: { + store: { + manifest, + baseDir: outputDir, + }, + }, }); expect(runtimeResult.manifestBaseDir).toBe(outputDir); expect(runtimeResult.frameworkResolved).toBe('bundled-framework'); expect(runtimeResult.frameworkStartEggMatches).toBe(true); expect(runtimeResult.controllerResolved).toBe('bundled-controller'); + expect(runtimeResult.loaderFSBaseDir).toBe(outputDir); + expect(runtimeResult.loaderFSUsesManifestStore).toBe(true); }); it('loads externalized package files via createRequire instead of static imports', async () => { @@ -416,7 +437,9 @@ export async function startEgg(options) { const worker = await fs.readFile(result.workerEntry, 'utf8'); expect(extractImports(worker).length).toBe(0); - expect(worker).toContain("startEgg({ baseDir: __outputDir, framework: __framework, mode: 'single' })"); + expect(worker).toContain( + "startEgg({ baseDir: __outputDir, framework: __framework, mode: 'single', loaderFS: __loaderFS })", + ); expect(worker).toContain('__EGG_BUNDLE_MODULE_LOADER__'); expect(worker).toContain('ManifestStore.setBundleStore'); }); diff --git a/tools/egg-bundler/test/__snapshots__/EntryGenerator.worker.canonical.snap b/tools/egg-bundler/test/__snapshots__/EntryGenerator.worker.canonical.snap index 4b2f8e79e3..c9e75769e8 100644 --- a/tools/egg-bundler/test/__snapshots__/EntryGenerator.worker.canonical.snap +++ b/tools/egg-bundler/test/__snapshots__/EntryGenerator.worker.canonical.snap @@ -2,7 +2,7 @@ /* eslint-disable */ import path from 'node:path'; -import { ManifestStore } from '@eggjs/core'; +import { ManifestLoaderFS, ManifestStore } from '@eggjs/core'; import type {} from '@eggjs/typings/global'; import { startEgg } from "egg"; import * as __frameworkModule from "egg"; @@ -99,12 +99,14 @@ for (const [appAbsRequest, targetRel] of __APP_RESOLVE_CACHE_ALIASES) { } } -ManifestStore.setBundleStore(ManifestStore.fromBundle(MANIFEST_DATA as any, __outputDir)); +const __bundleManifestStore = ManifestStore.fromBundle(MANIFEST_DATA as any, __outputDir); +const __loaderFS = new ManifestLoaderFS(__bundleManifestStore); +ManifestStore.setBundleStore(__bundleManifestStore); globalThis.__EGG_BUNDLE_MODULE_LOADER__ = (filepath) => { return __getBundleMap(filepath); }; -startEgg({ baseDir: __outputDir, framework: __framework, mode: 'single' }).then((app) => { +startEgg({ baseDir: __outputDir, framework: __framework, mode: 'single', loaderFS: __loaderFS }).then((app) => { const port = process.env.PORT || app.config.cluster?.listen?.port || 7001; app.listen(port, () => { // eslint-disable-next-line no-console From 8cef6227ab1cd0739bb71b9700538f31ad2939b9 Mon Sep 17 00:00:00 2001 From: killa Date: Sun, 10 May 2026 11:44:14 +0800 Subject: [PATCH 3/4] perf(core): index manifest loader fs paths --- packages/core/src/loader/egg_loader.ts | 25 ++++--- packages/core/src/loader/loader_fs.ts | 94 ++++++++++++++------------ 2 files changed, 65 insertions(+), 54 deletions(-) diff --git a/packages/core/src/loader/egg_loader.ts b/packages/core/src/loader/egg_loader.ts index fcc0dfb3b0..9726452918 100644 --- a/packages/core/src/loader/egg_loader.ts +++ b/packages/core/src/loader/egg_loader.ts @@ -1801,14 +1801,17 @@ export class EggLoader { * generation time. */ #collectConventionalDynamicFiles(manifest: StartupManifest): void { + const resolveCacheValues = new Set( + Object.values(manifest.resolveCache).filter((target): target is string => typeof target === 'string'), + ); for (const unit of this.getLoadUnits()) { - this.#collectConventionFile(manifest, path.join(unit.path, 'package.json')); + this.#collectConventionFile(manifest, path.join(unit.path, 'package.json'), resolveCacheValues); for (const load of CONVENTIONAL_MANIFEST_LOADS) { const target = path.join(unit.path, ...load.path); if (load.type === 'resolve') { - this.#collectConventionResolve(manifest, target); + this.#collectConventionResolve(manifest, target, resolveCacheValues); } else if ('extensionlessResolve' in load && load.extensionlessResolve) { - this.#collectConventionFileResolves(manifest, target); + this.#collectConventionFileResolves(manifest, target, resolveCacheValues); } else { this.#collectConventionFileDiscovery(manifest, target); } @@ -1816,21 +1819,25 @@ export class EggLoader { } } - #collectConventionResolve(manifest: StartupManifest, request: string): void { + #collectConventionResolve(manifest: StartupManifest, request: string, resolveCacheValues: Set): void { const requestKey = this.#toManifestRel(request); if (Object.hasOwn(manifest.resolveCache, requestKey)) return; const resolved = this.#doResolveModule(request); - manifest.resolveCache[requestKey] = resolved ? this.#toManifestRel(resolved) : null; + const resolvedKey = resolved ? this.#toManifestRel(resolved) : null; + manifest.resolveCache[requestKey] = resolvedKey; + if (resolvedKey) { + resolveCacheValues.add(resolvedKey); + } } - #collectConventionFileResolves(manifest: StartupManifest, directory: string): void { + #collectConventionFileResolves(manifest: StartupManifest, directory: string, resolveCacheValues: Set): void { const files = this.#collectConventionFileDiscovery(manifest, directory); for (const file of files) { const ext = path.extname(file); if (!ext) continue; const request = path.join(directory, file.slice(0, -ext.length)); - this.#collectConventionResolve(manifest, request); + this.#collectConventionResolve(manifest, request, resolveCacheValues); } } @@ -1845,9 +1852,9 @@ export class EggLoader { return manifest.fileDiscovery[dirKey]; } - #collectConventionFile(manifest: StartupManifest, filepath: string): void { + #collectConventionFile(manifest: StartupManifest, filepath: string, resolveCacheValues: Set): void { const fileKey = this.#toManifestRel(filepath); - if (Object.values(manifest.resolveCache).includes(fileKey)) return; + if (resolveCacheValues.has(fileKey)) return; if (!fs.existsSync(filepath) || !fs.statSync(filepath).isFile()) return; const dirKey = this.#toManifestRel(path.dirname(filepath)); diff --git a/packages/core/src/loader/loader_fs.ts b/packages/core/src/loader/loader_fs.ts index f8e3739d97..f743e27ddf 100644 --- a/packages/core/src/loader/loader_fs.ts +++ b/packages/core/src/loader/loader_fs.ts @@ -49,10 +49,33 @@ export class RealLoaderFS implements LoaderFS { export class ManifestLoaderFS implements LoaderFS { readonly #manifest: ManifestStore; readonly #fallback: LoaderFS; + readonly #manifestFiles: Set; + readonly #manifestDirectories: Set; + readonly #resolveCacheTargets: Set; constructor(manifest: ManifestStore, fallback: LoaderFS = new RealLoaderFS()) { this.#manifest = manifest; this.#fallback = fallback; + this.#resolveCacheTargets = new Set( + Object.values(manifest.data.resolveCache).filter((target): target is string => typeof target === 'string'), + ); + + const manifestFiles = new Set(); + const manifestDirectories = new Set(); + for (const [dir, files] of Object.entries(manifest.data.fileDiscovery)) { + addManifestDirectory(manifestDirectories, dir); + for (const file of files) { + const fullRel = path.posix.join(dir, file); + manifestFiles.add(fullRel); + addManifestDirectory(manifestDirectories, path.posix.dirname(fullRel)); + } + } + for (const target of this.#resolveCacheTargets) { + manifestFiles.add(target); + addManifestDirectory(manifestDirectories, path.posix.dirname(target)); + } + this.#manifestFiles = manifestFiles; + this.#manifestDirectories = manifestDirectories; } exists(filepath: string): boolean { @@ -142,15 +165,10 @@ export class ManifestLoaderFS implements LoaderFS { } #resolveFromFileDiscovery(rel: string): string | undefined { - let matchedDir: string | undefined; - for (const dir of Object.keys(this.#manifest.data.fileDiscovery)) { - if ((rel === dir || rel.startsWith(dir + '/')) && (!matchedDir || dir.length > matchedDir.length)) { - matchedDir = dir; - } - } - if (!matchedDir || rel === matchedDir) return; + const matchedDir = this.#nearestManifestDiscoveryDir(rel); + if (matchedDir === undefined || rel === matchedDir) return; - const request = rel.slice(matchedDir.length + 1); + const request = matchedDir === '' ? rel : rel.slice(matchedDir.length + 1); for (const file of this.#manifest.data.fileDiscovery[matchedDir]) { if (file === request) { return path.posix.join(matchedDir, file); @@ -166,41 +184,24 @@ export class ManifestLoaderFS implements LoaderFS { } } - #isManifestFile(rel: string): boolean { - for (const [dir, files] of Object.entries(this.#manifest.data.fileDiscovery)) { - if (files.includes(path.posix.relative(dir, rel))) { - return true; + #nearestManifestDiscoveryDir(rel: string): string | undefined { + let current = path.posix.dirname(rel); + while (true) { + const dir = current === '.' ? '' : current; + if (Object.hasOwn(this.#manifest.data.fileDiscovery, dir)) { + return dir; } + if (dir === '') return; + current = path.posix.dirname(dir); } - return Object.values(this.#manifest.data.resolveCache).includes(rel); } - #isManifestDirectory(rel: string): boolean { - if (rel === '') { - return this.#hasManifestData(); - } - if (Object.hasOwn(this.#manifest.data.fileDiscovery, rel)) { - return true; - } - - for (const [dir, files] of Object.entries(this.#manifest.data.fileDiscovery)) { - if (dir.startsWith(rel + '/')) { - return true; - } - for (const file of files) { - const fullRel = path.posix.join(dir, file); - if (path.posix.dirname(fullRel) === rel || fullRel.startsWith(rel + '/')) { - return true; - } - } - } + #isManifestFile(rel: string): boolean { + return this.#manifestFiles.has(rel); + } - for (const target of Object.values(this.#manifest.data.resolveCache)) { - if (target && (path.posix.dirname(target) === rel || target.startsWith(rel + '/'))) { - return true; - } - } - return false; + #isManifestDirectory(rel: string): boolean { + return this.#manifestDirectories.has(rel); } #listManifestFilesUnder(cwdRel: string): string[] | undefined { @@ -242,13 +243,6 @@ export class ManifestLoaderFS implements LoaderFS { return [...new Set([rel, normalizePath(abs)])]; } - #hasManifestData(): boolean { - return ( - Object.keys(this.#manifest.data.fileDiscovery).length > 0 || - Object.keys(this.#manifest.data.resolveCache).some((key) => this.#manifest.data.resolveCache[key] !== null) - ); - } - #toRelative(filepath: string): string { const rel = path.isAbsolute(filepath) ? path.relative(this.#manifest.baseDir, filepath) : filepath; return normalizePath(rel); @@ -268,6 +262,16 @@ function normalizePath(filepath: string): string { return filepath.replaceAll(path.sep, '/'); } +function addManifestDirectory(directories: Set, dir: string): void { + let current = dir === '.' ? '' : dir; + while (true) { + directories.add(current); + if (current === '') return; + const parent = path.posix.dirname(current); + current = parent === '.' ? '' : parent; + } +} + function relativePrefix(cwdRel: string, dirRel: string): string | undefined { if (cwdRel === '') return dirRel; if (dirRel === cwdRel) return ''; From e153fd4a245c1233e257e822901b0a11bb2aefef Mon Sep 17 00:00:00 2001 From: killa Date: Sun, 10 May 2026 12:48:33 +0800 Subject: [PATCH 4/4] fix(core): keep default loader fs real --- packages/core/src/loader/egg_loader.ts | 3 --- packages/core/test/egg.test.ts | 8 +++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/core/src/loader/egg_loader.ts b/packages/core/src/loader/egg_loader.ts index 9726452918..0f57e20fcc 100644 --- a/packages/core/src/loader/egg_loader.ts +++ b/packages/core/src/loader/egg_loader.ts @@ -179,9 +179,6 @@ export class EggLoader { this.manifest = ManifestStore.load(this.options.baseDir, this.serverEnv, this.serverScope) ?? ManifestStore.createCollector(this.options.baseDir); - if (!this.options.loaderFS && !(this.loaderFS instanceof ManifestLoaderFS)) { - this.loaderFS = new ManifestLoaderFS(this.manifest, this.loaderFS); - } } get app(): EggCore { diff --git a/packages/core/test/egg.test.ts b/packages/core/test/egg.test.ts index eb3b3bacb8..238445fdf3 100644 --- a/packages/core/test/egg.test.ts +++ b/packages/core/test/egg.test.ts @@ -9,7 +9,7 @@ import coffee from 'coffee'; import { mm } from 'mm'; import { describe, it, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; -import { EggCore, RealLoaderFS } from '../src/index.js'; +import { EggCore, ManifestLoaderFS, RealLoaderFS } from '../src/index.js'; import { createApp, getFilepath, type Application } from './helper.js'; describe('test/egg.test.ts', () => { @@ -60,6 +60,12 @@ describe('test/egg.test.ts', () => { assert.equal(app.loader.loaderFS, loaderFS); }); + it('should use RealLoaderFS by default for non-bundled runtime', () => { + app = new EggCore(); + assert.equal(app.loader.loaderFS instanceof RealLoaderFS, true); + assert.equal(app.loader.loaderFS instanceof ManifestLoaderFS, false); + }); + it('should not set value expect for application and agent', () => { assert.throws( () =>