fix(schema-compiler): invalidate Jinja render cache when imported macro file changes (cube-js#10818)

* fix(schema-compiler): CUB-2357 invalidate Jinja render cache when imported macro file changes

* fix(schema-compiler): use \0 separator in Jinja cache key for consistency

Author: igorlukanin

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -28,6 +28,7 @@ const NATIVE_IS_SUPPORTED = isNativeSupported();
 const moduleFileCache = {};
 
 const JINJA_SYNTAX = /{%|%}|{{|}}/ig;
+const JINJA_MACRO_DEFINITION = /{%[-+]?\s*macro\s/;
 
 const getThreadsCount = () => {
   const envThreads = getEnv('transpilationWorkerThreadsCount');
@@ -100,6 +101,7 @@ export type TranspileOptions = {
   compilerId?: string;
   stage?: 0 | 1 | 2 | 3;
   jinjaUsed?: boolean;
+  jinjaMacrosFingerprint?: string;
 };
 
 export type CompileStage = 0 | 1 | 2 | 3;
@@ -280,6 +282,8 @@ export class DataSchemaCompiler {
       this.loadJinjaTemplates(jinjaTemplatedFiles);
     }
 
+    const jinjaMacrosFingerprint = DataSchemaCompiler.computeJinjaMacrosFingerprint(jinjaTemplatedFiles);
+
     const errorsReport = new ErrorReporter(null, [], this.errorReportOptions);
     this.errorsReporter = errorsReport;
 
@@ -328,11 +332,11 @@ export class DataSchemaCompiler {
         }
 
         const jinjaFilesTasks = jinjaTemplatedFiles
-          .map(f => this.transpileJinjaFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames }));
+          .map(f => this.transpileJinjaFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames, jinjaMacrosFingerprint }));
 
         results = (await Promise.all([...jsFilesTasks, ...yamlFilesTasks, ...jinjaFilesTasks])).flat();
       } else {
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames, jinjaMacrosFingerprint })));
       }
 
       return results.filter(f => !!f) as FileContent[];
@@ -576,6 +580,33 @@ export class DataSchemaCompiler {
     });
   }
 
+  /**
+   * Macro files are hidden dependencies of any cube file that imports them —
+   * minijinja resolves `{% import %}` lazily against its template store, so
+   * the per-file Jinja render cache must be invalidated when *any* macro file
+   * changes. Hashing all macro files together rather than tracking per-cube
+   * imports keeps the implementation simple at the cost of over-invalidating
+   * when macro edits happen (which is rare). CUB-2357.
+   */
+  private static computeJinjaMacrosFingerprint(files: FileContent[]): string {
+    const macroFiles = files
+      .filter((f) => JINJA_MACRO_DEFINITION.test(f.content))
+      .sort((a, b) => a.fileName.localeCompare(b.fileName));
+
+    if (macroFiles.length === 0) {
+      return '';
+    }
+
+    const hash = crypto.createHash('md5');
+    for (const f of macroFiles) {
+      hash.update(f.fileName);
+      hash.update('\0');
+      hash.update(f.content);
+      hash.update('\0');
+    }
+    return hash.digest('hex');
+  }
+
   private prepareTranspileSymbols() {
     const cubeNames: string[] = this.cubeDictionary.cubeNames();
     // We need only cubes and all its member names for transpiling.
@@ -802,7 +833,11 @@ export class DataSchemaCompiler {
     errorsReport: ErrorReporter,
     options: TranspileOptions
   ): Promise<(FileContent | undefined)> {
-    const cacheKey = crypto.createHash('md5').update(file.content).digest('hex');
+    const cacheKey = crypto.createHash('md5')
+      .update(file.content)
+      .update('\0')
+      .update(options.jinjaMacrosFingerprint || '')
+      .digest('hex');
 
     let renderedFileContent: string;
 

packages/cubejs-schema-compiler/test/integration/postgres/jinja-macro-cache.test.ts

@@ -0,0 +1,98 @@
+import { LRUCache } from 'lru-cache';
+import { FileContent, isNativeSupported } from '@cubejs-backend/shared';
+
+import { prepareCompiler } from '../../../src/compiler/PrepareCompiler';
+
+const suite = isNativeSupported() === true ? describe : xdescribe;
+
+const cubeFile = (name: string, extraDimsBlock: string): string => `{% import 'macros.yml' as macros %}
+
+cubes:
+  - name: ${name}
+    sql: >
+      SELECT 1 AS id
+
+    dimensions:
+      - name: id
+        sql: id
+        type: number
+        primary_key: true
+${extraDimsBlock}
+`;
+
+const macroFile = (dimensionName: string) => `{% macro dimensions() %}
+      - name: ${dimensionName}
+        sql: ${dimensionName}
+        type: string
+{% endmacro %}
+`;
+
+async function compileWith(files: FileContent[], compiledJinjaCache: LRUCache<string, string>) {
+  const repo = {
+    localPath: () => __dirname,
+    dataSchemaFiles: () => Promise.resolve(files),
+  };
+
+  const { compiler, metaTransformer } = prepareCompiler(repo, {
+    adapter: 'postgres',
+    compiledJinjaCache,
+  } as any);
+
+  await compiler.compile();
+
+  return { metaTransformer };
+}
+
+function dimensionNames(metaTransformer: any, cubeName: string): string[] {
+  const cube = metaTransformer.cubes.find((c: any) => c.config.name === cubeName);
+  return cube.config.dimensions.map((d: any) => d.name);
+}
+
+suite('Jinja macro cache invalidation', () => {
+  it('invalidates the cube file render cache when a macro file changes (CUB-2357)', async () => {
+    const sharedCache = new LRUCache<string, string>({ max: 250 });
+
+    const filesV1: FileContent[] = [
+      { fileName: 'orders.yml', content: cubeFile('orders', '{{ macros.dimensions() }}') },
+      { fileName: 'macros.yml', content: macroFile('status') },
+    ];
+
+    const v1 = await compileWith(filesV1, sharedCache);
+    expect(dimensionNames(v1.metaTransformer, 'orders')).toEqual(['orders.id', 'orders.status']);
+
+    const filesV2: FileContent[] = [
+      filesV1[0],
+      { fileName: 'macros.yml', content: macroFile('priority') },
+    ];
+
+    const v2 = await compileWith(filesV2, sharedCache);
+    expect(dimensionNames(v2.metaTransformer, 'orders')).toEqual(['orders.id', 'orders.priority']);
+  });
+
+  it('reuses the render cache for unchanged cube files when a sibling cube file changes', async () => {
+    const sharedCache = new LRUCache<string, string>({ max: 250 });
+
+    const filesV1: FileContent[] = [
+      { fileName: 'orders.yml', content: cubeFile('orders', '') },
+      { fileName: 'products.yml', content: cubeFile('products', '') },
+      { fileName: 'macros.yml', content: macroFile('unused') },
+    ];
+    await compileWith(filesV1, sharedCache);
+
+    const cacheSizeAfterFirstCompile = sharedCache.size;
+
+    const filesV2: FileContent[] = [
+      {
+        fileName: 'orders.yml',
+        content: cubeFile('orders', '      - name: status\n        sql: status\n        type: string\n'),
+      },
+      filesV1[1],
+      filesV1[2],
+    ];
+    await compileWith(filesV2, sharedCache);
+
+    // Only the changed orders.yml should miss the cache; products.yml and
+    // macros.yml are byte-identical and the macros fingerprint is unchanged.
+    expect(sharedCache.size).toBe(cacheSizeAfterFirstCompile + 1);
+  });
+});