feat(nf): Added hash to metadata files to avoid collision

Author: Aukevanoost

libs/native-federation-core/src/lib/core/build-for-federation.ts

@@ -14,7 +14,7 @@ import { FederationOptions } from './federation-options';
 import { writeFederationInfo } from './write-federation-info';
 import { writeImportMap } from './write-import-map';
 import { logger } from '../utils/logger';
-import { getCachePath } from './get-cache';
+import { getCachePath } from './bundle-caching';
 import { normalizeFilename } from '../utils/normalize';
 
 export interface BuildParams {
@@ -82,7 +82,7 @@ export async function buildForFederation(
         fedOptions,
         externals,
         'browser',
-        { pathToCache, metaDataFile: 'meta-browser-shared.json' }
+        { pathToCache, bundleName: 'browser-shared' }
       );
 
       logger.measure(
@@ -101,7 +101,7 @@ export async function buildForFederation(
         fedOptions,
         externals,
         'node',
-        { pathToCache, metaDataFile: 'meta-node-shared.json' }
+        { pathToCache, bundleName: 'node-shared' }
       );
       logger.measure(
         start,
@@ -204,7 +204,7 @@ async function bundleSeparate(
         platform,
         {
           pathToCache,
-          metaDataFile: `meta-${platform}-${normalizeFilename(key)}.json`,
+          bundleName: `${platform}-${normalizeFilename(key)}`,
         }
       );
     }
@@ -223,7 +223,6 @@ function splitShared(
   const separateServer: Record<string, NormalizedSharedConfig> = {};
 
   for (const key in shared) {
-    // if (cachedSharedPackages.has(key)) continue;
     const obj = shared[key];
     if (obj.platform === 'node' && obj.build === 'default') {
       sharedServer[key] = obj;

libs/native-federation-core/src/lib/core/bundle-caching.ts

@@ -0,0 +1,112 @@
+import path from 'path';
+import fs from 'fs';
+import crypto from 'crypto';
+import { NormalizedSharedConfig } from '../config/federation-config';
+import { SharedInfo } from '@softarc/native-federation-runtime';
+import { logger } from '../utils/logger';
+
+export const getCachePath = (workspaceRoot: string, project: string) =>
+  path.join(workspaceRoot, 'node_modules/.cache/native-federation', project);
+
+export const getFilename = (checksum: string, title: string) => {
+  return `${title}-meta-${checksum.substring(0, 10)}.json`;
+};
+
+export const getChecksum = (
+  shared: Record<string, NormalizedSharedConfig>
+): string => {
+  const denseExternals = Object.keys(shared)
+    .sort()
+    .reduce((clean, external) => {
+      return (
+        clean +
+        ':' +
+        external +
+        (shared[external].version ? `@${shared[external].version}` : '')
+      );
+    }, 'deps');
+
+  return crypto.createHash('sha256').update(denseExternals).digest('hex');
+};
+
+export const cacheEntry = (pathToCache: string, fileName: string) => ({
+  getMetadata: (
+    checksum: string
+  ):
+    | {
+        checksum: string;
+        externals: SharedInfo[];
+        files: string[];
+      }
+    | undefined => {
+    const metadataFile = path.join(pathToCache, fileName);
+    if (!fs.existsSync(pathToCache) || !fs.existsSync(metadataFile))
+      return undefined;
+
+    const cachedResult: {
+      checksum: string;
+      externals: SharedInfo[];
+      files: string[];
+    } = JSON.parse(fs.readFileSync(metadataFile, 'utf-8'));
+    if (cachedResult.checksum !== checksum) return undefined;
+    return cachedResult;
+  },
+  persist: (payload: {
+    checksum: string;
+    externals: SharedInfo[];
+    files: string[];
+  }) => {
+    fs.writeFileSync(
+      path.join(pathToCache, fileName),
+      JSON.stringify(payload),
+      'utf-8'
+    );
+  },
+  copyFiles: (fullOutputPath: string) => {
+    const metadataFile = path.join(pathToCache, fileName);
+    if (!fs.existsSync(metadataFile))
+      throw new Error(
+        'Error copying artifacts to dist, metadata file could not be found.'
+      );
+
+    const cachedResult: {
+      externals: SharedInfo[];
+      files: string[];
+    } = JSON.parse(fs.readFileSync(metadataFile, 'utf-8'));
+
+    fs.mkdirSync(path.dirname(fullOutputPath), { recursive: true });
+
+    cachedResult.files.forEach((file) => {
+      const cachedFile = path.join(pathToCache, file);
+      const distFileName = path.join(fullOutputPath, file);
+
+      if (fs.existsSync(cachedFile)) {
+        fs.copyFileSync(cachedFile, distFileName);
+      }
+    });
+  },
+  clear: () => {
+    const metadataFile = path.join(pathToCache, fileName);
+    if (!fs.existsSync(pathToCache)) {
+      fs.mkdirSync(pathToCache, { recursive: true });
+      return;
+    }
+    if (!fs.existsSync(metadataFile)) {
+      logger.warn(
+        `Could not purge cache, metadata file '${fileName}' could not be found.`
+      );
+      return;
+    }
+
+    const cachedResult: {
+      checksum: string;
+      externals: SharedInfo[];
+      files: string[];
+    } = JSON.parse(fs.readFileSync(metadataFile, 'utf-8'));
+
+    cachedResult.files.forEach((file) => {
+      const cachedFile = path.join(pathToCache, file);
+      if (fs.existsSync(cachedFile)) fs.unlinkSync(cachedFile);
+    });
+  },
+});

libs/native-federation-core/src/lib/core/bundle-shared.ts

@@ -17,47 +17,38 @@ import {
   isSourceFile,
   rewriteChunkImports,
 } from '../utils/rewrite-chunk-imports';
-import {
-  copyCacheToDist,
-  getCachedMetadata,
-  getChecksum,
-  purgeCacheFolder,
-  storeCachedMetadata,
-} from './get-cache';
+import { cacheEntry, getChecksum, getFilename } from './bundle-caching';
 
 export async function bundleShared(
   sharedBundles: Record<string, NormalizedSharedConfig>,
   config: NormalizedFederationConfig,
   fedOptions: FederationOptions,
   externals: string[],
   platform: 'browser' | 'node' = 'browser',
-  cache: { pathToCache: string; metaDataFile: string }
+  cacheOptions: { pathToCache: string; bundleName: string }
 ): Promise<Array<SharedInfo>> {
   const checksum = getChecksum(sharedBundles);
   const folder = fedOptions.packageJson
     ? path.dirname(fedOptions.packageJson)
     : fedOptions.workspaceRoot;
 
-  const sharedPackageInfoCache = getCachedMetadata(
-    cache.pathToCache,
-    cache.metaDataFile,
-    checksum
+  const bundleCache = cacheEntry(
+    cacheOptions.pathToCache,
+    getFilename(checksum, cacheOptions.bundleName)
   );
 
-  if (sharedPackageInfoCache) {
+  const cacheMetadata = bundleCache.getMetadata(checksum);
+  if (cacheMetadata) {
     logger.info(
-      `Checksum of ${cache.metaDataFile} matched, Skipped artifact bundling`
+      `Checksum of ${cacheOptions.bundleName} matched, Skipped artifact bundling`
     );
-    copyCacheToDist(
-      cache.pathToCache,
-      cache.metaDataFile,
+    bundleCache.copyFiles(
       path.join(fedOptions.workspaceRoot, fedOptions.outputPath)
     );
-    return sharedPackageInfoCache;
+    return cacheMetadata.externals;
   }
 
-  purgeCacheFolder(cache.pathToCache, cache.metaDataFile);
-  fs.mkdirSync(cache.pathToCache, { recursive: true });
+  bundleCache.clear();
 
   const inferredPackageInfos = Object.keys(sharedBundles)
     .filter((packageName) => !sharedBundles[packageName].packageInfo)
@@ -93,7 +84,7 @@ export async function bundleShared(
     path.join(fullOutputPath, ep.outName)
   );
   const entryPoints = allEntryPoints.filter(
-    (ep) => !fs.existsSync(path.join(cache.pathToCache, ep.outName))
+    (ep) => !fs.existsSync(path.join(cacheOptions.pathToCache, ep.outName))
   );
 
   if (entryPoints.length > 0) {
@@ -120,7 +111,7 @@ export async function bundleShared(
       entryPoints,
       tsConfigPath: fedOptions.tsConfig,
       external: [...additionalExternals, ...externals],
-      outdir: cache.pathToCache,
+      outdir: cacheOptions.pathToCache,
       mappedPaths: config.sharedMappings,
       dev: fedOptions.dev,
       kind: 'shared-package',
@@ -130,7 +121,7 @@ export async function bundleShared(
     });
 
     const cachedFiles = bundleResult.map((br) => path.basename(br.fileName));
-    rewriteImports(cachedFiles, cache.pathToCache);
+    rewriteImports(cachedFiles, cacheOptions.pathToCache);
   } catch (e) {
     logger.error('Error bundling shared npm package ');
     if (e instanceof Error) {
@@ -165,12 +156,7 @@ export async function bundleShared(
 
   const outFileNames = [...expectedResults];
 
-  const result = buildResult(
-    packageInfos,
-    sharedBundles,
-    outFileNames,
-    fedOptions
-  );
+  const result = buildResult(packageInfos, sharedBundles, outFileNames);
 
   // TODO: Decide whether/when to add .map files
   const chunks = bundleResult.filter(
@@ -181,15 +167,13 @@ export async function bundleShared(
 
   addChunksToResult(chunks, result, fedOptions.dev);
 
-  storeCachedMetadata(cache.pathToCache, cache.metaDataFile, {
+  bundleCache.persist({
     checksum,
     externals: result,
     files: bundleResult.map((r) => r.fileName.split('/').pop() ?? r.fileName),
   });
 
-  copyCacheToDist(
-    cache.pathToCache,
-    cache.metaDataFile,
+  bundleCache.copyFiles(
     path.join(fedOptions.workspaceRoot, fedOptions.outputPath)
   );
 
@@ -220,28 +204,10 @@ function createOutName(
   return outName;
 }
 
-function createCacheFileName(
-  configState: string,
-  sharedBundles: Record<string, NormalizedSharedConfig>,
-  fedOptions: FederationOptions,
-  cachePath: string,
-  platform: string
-) {
-  const resultCacheState = configState + JSON.stringify(sharedBundles);
-  const resultHash = calcHash(resultCacheState);
-  const dev = fedOptions.dev ? '-dev' : '';
-  const resultCacheFile = path.join(
-    cachePath,
-    'result-' + resultHash + '-' + platform + dev + '.json'
-  );
-  return resultCacheFile;
-}
-
 function buildResult(
   packageInfos: PackageInfo[],
   sharedBundles: Record<string, NormalizedSharedConfig>,
-  outFileNames: string[],
-  fedOptions: FederationOptions
+  outFileNames: string[]
 ) {
   return packageInfos.map((pi) => {
     const shared = sharedBundles[pi.packageName];