Fixes for projects that use cds-indexer

extractors/cds/tools/src/cds/indexer.ts

@@ -198,24 +198,53 @@ export function orchestrateCdsIndexer(
 
     summary.projectsRequiringIndexer++;
     const cacheDir = projectCacheDirMap.get(projectDir);
+
+    // Install the project's full dependencies BEFORE running cds-indexer.
+    //
+    // The indexer loads the project's cds runtime to walk the model and decide
+    // which `.cds` files to include in the generated `index.cds` files. If the
+    // project's plugins (e.g. `@sap/cds-shim`, `@sap/cds-mtxs`, `@cap-js/hana`,
+    // file-based local plugins) are not yet installed, those plugins never
+    // register their hooks, and the indexer sees a stripped-down model. The
+    // resulting `index.cds` files then fail to transitively include every file
+    // the project's annotations reference, and later `cds compile` fails with
+    // "Artifact not found" errors.
+    //
+    // The cache directory only contains `@sap/cds`, `@sap/cds-dk`, and
+    // `@sap/cds-indexer`, which is not enough for projects that reference
+    // additional CDS packages — hence the project-local install.
+    const installResult = projectInstallDependencies(project, sourceRoot);
+
+    // Record the install on the project's retry status so the retry path
+    // (see needsFullDependencyInstallation) doesn't run a second redundant
+    // `npm install` after compilation. Mirrors the bookkeeping done in
+    // retry.ts when the retry path itself drives the install.
+    project.retryStatus ??= {
+      fullDependenciesInstalled: false,
+      tasksRequiringRetry: 0,
+      tasksRetried: 0,
+      installationErrors: [],
+    };
+
+    if (installResult.success) {
+      project.retryStatus.fullDependenciesInstalled = true;
+      dependencyGraph.retryStatus.projectsWithFullDependencies.add(projectDir);
+    } else {
+      project.retryStatus.installationErrors = [
+        ...(project.retryStatus.installationErrors ?? []),
+        installResult.error ?? 'Unknown installation error',
+      ];
+      cdsExtractorLog(
+        'warn',
+        `Full dependency installation failed for project '${projectDir}' before cds-indexer run: ${installResult.error ?? 'unknown error'}. Continuing with indexer attempt — it may still produce useful output.`,
+      );
+    }
+
     const result = runCdsIndexer(project, sourceRoot, cacheDir);
     summary.results.push(result);
 
     if (result.success) {
       summary.successfulRuns++;
-
-      // Install the project's full dependencies so the CDS compiler can
-      // resolve all CDS model imports (e.g. `@sap/cds-shim`) during
-      // compilation.  The cache directory only contains `@sap/cds`,
-      // `@sap/cds-dk`, and `@sap/cds-indexer`, which is not enough for
-      // projects that reference additional CDS packages.
-      const installResult = projectInstallDependencies(project, sourceRoot);
-      if (!installResult.success) {
-        cdsExtractorLog(
-          'warn',
-          `Full dependency installation failed for project '${projectDir}' after successful cds-indexer run: ${installResult.error ?? 'unknown error'}`,
-        );
-      }
     } else {
       summary.failedRuns++;
 

extractors/cds/tools/src/packageManager/projectInstaller.ts

@@ -1,6 +1,7 @@
 /** Full dependency installation utilities for retry scenarios. */
 
 import { execFileSync } from 'child_process';
+import { existsSync } from 'fs';
 import { join } from 'path';
 
 import { npmExecutable, getPlatformInfo } from '../environment';
@@ -65,10 +66,24 @@ export function projectInstallDependencies(
       return result;
     }
 
-    // Install dependencies using npm in the project's directory
+    // Install dependencies using npm in the project's directory.
+    //
+    // Prefer `npm ci` when a lockfile exists: it installs exactly the versions
+    // the project pinned, matching what `npm ci` would do in CI. Fall back to
+    // `npm install` only when there is no lockfile.
+    //
+    // We deliberately do NOT pass `--ignore-scripts`. Many cds plugins
+    // (e.g. file-based local plugins, and packages whose `cds-plugin.js`
+    // entries get wired via `postinstall`/`prepare`) need their lifecycle
+    // scripts to run for the cds runtime to discover them. Without those
+    // scripts, the indexer walks an under-populated model and the resulting
+    // `index.cds` files omit artifacts that annotation files reference,
+    // causing later `cds compile` to fail with "Artifact not found".
+    const hasLockfile = existsSync(join(projectPath, 'package-lock.json'));
+    const installSubcommand = hasLockfile ? 'ci' : 'install';
     cdsExtractorLog(
       'info',
-      `Installing full dependencies for project ${project.projectDir} in project's node_modules`,
+      `Installing full dependencies for project ${project.projectDir} via 'npm ${installSubcommand}' in project's node_modules`,
     );
 
     try {
@@ -78,14 +93,7 @@ export function projectInstallDependencies(
         // (e.g. engines.node ^18) which would otherwise abort the install on newer Node.
         // npm's default is non-strict; we make that explicit so the project's .npmrc
         // can't flip it on and break the retry path.
-        [
-          'install',
-          '--engine-strict=false',
-          '--ignore-scripts',
-          '--quiet',
-          '--no-audit',
-          '--no-fund',
-        ],
+        [installSubcommand, '--engine-strict=false', '--quiet', '--no-audit', '--no-fund'],
         {
           cwd: projectPath,
           stdio: 'inherit',

extractors/cds/tools/test/src/cds/indexer.test.ts

@@ -494,7 +494,7 @@ describe('cds/indexer', () => {
       );
     });
 
-    it('should install full project dependencies after successful cds-indexer run', () => {
+    it('should install full project dependencies before running cds-indexer', () => {
       (childProcess.spawnSync as jest.Mock).mockReturnValue({
         status: 0,
         stdout: Buffer.from(''),
@@ -516,7 +516,10 @@ describe('cds/indexer', () => {
       expect(projectInstaller.projectInstallDependencies).toHaveBeenCalledWith(project, '/source');
     });
 
-    it('should not install full project dependencies when cds-indexer fails', () => {
+    it('should still install full project dependencies even when cds-indexer subsequently fails', () => {
+      // Install runs BEFORE the indexer now, so an indexer failure can no longer
+      // gate the install. We still want the install to have happened so that
+      // downstream compilation has a fully-populated node_modules to work with.
       (childProcess.spawnSync as jest.Mock).mockReturnValue({
         status: 1,
         stdout: Buffer.from(''),
@@ -535,7 +538,7 @@ describe('cds/indexer', () => {
 
       orchestrateCdsIndexer(graph, '/source', new Map());
 
-      expect(projectInstaller.projectInstallDependencies).not.toHaveBeenCalled();
+      expect(projectInstaller.projectInstallDependencies).toHaveBeenCalledWith(project, '/source');
     });
 
     it('should not install full project dependencies for projects without cds-indexer', () => {
@@ -550,7 +553,7 @@ describe('cds/indexer', () => {
       expect(projectInstaller.projectInstallDependencies).not.toHaveBeenCalled();
     });
 
-    it('should continue even when full dependency installation fails after indexer success', () => {
+    it('should continue even when full dependency installation fails before the indexer runs', () => {
       (childProcess.spawnSync as jest.Mock).mockReturnValue({
         status: 0,
         stdout: Buffer.from(''),

extractors/cds/tools/test/src/packageManager/projectInstaller.test.ts

@@ -1,6 +1,7 @@
 /** Tests for CDS compiler installer functionality */
 
 import { execFileSync } from 'child_process';
+import { existsSync } from 'fs';
 
 import type { CdsProject } from '../../../src/cds/parser';
 import { getPlatformInfo } from '../../../src/environment';
@@ -11,14 +12,19 @@ import {
 
 // Mock all external dependencies
 jest.mock('child_process');
+jest.mock('fs');
 
 const mockExecFileSync = execFileSync as jest.MockedFunction<typeof execFileSync>;
+const mockExistsSync = existsSync as jest.MockedFunction<typeof existsSync>;
 
 describe('CDS Compiler Installer', () => {
   const mockSourceRoot = '/test/source';
 
   beforeEach(() => {
     jest.clearAllMocks();
+    // Default: no package-lock.json present, so projectInstallDependencies
+    // falls through to 'npm install'. Tests that need 'npm ci' override this.
+    mockExistsSync.mockReturnValue(false);
     // Mock Date.now to return a consistent timestamp for testing
     jest.spyOn(Date, 'now').mockReturnValue(1640995200000);
     // Mock process.hrtime.bigint for performance timing
@@ -65,14 +71,28 @@ describe('CDS Compiler Installer', () => {
 
       expect(mockExecFileSync).toHaveBeenCalledWith(
         'npm',
-        [
-          'install',
-          '--engine-strict=false',
-          '--ignore-scripts',
-          '--quiet',
-          '--no-audit',
-          '--no-fund',
-        ],
+        ['install', '--engine-strict=false', '--quiet', '--no-audit', '--no-fund'],
+        {
+          cwd: '/test/source/test-project',
+          shell: getPlatformInfo().isWindows,
+          stdio: 'inherit',
+          timeout: 120000,
+        },
+      );
+    });
+
+    it("should use 'npm ci' when a package-lock.json exists in the project", () => {
+      const project = createMockProject();
+
+      mockExistsSync.mockReturnValue(true);
+      mockExecFileSync.mockReturnValue(Buffer.from(''));
+
+      const result = projectInstallDependencies(project, mockSourceRoot);
+
+      expect(result.success).toBe(true);
+      expect(mockExecFileSync).toHaveBeenCalledWith(
+        'npm',
+        ['ci', '--engine-strict=false', '--quiet', '--no-audit', '--no-fund'],
         {
           cwd: '/test/source/test-project',
           shell: getPlatformInfo().isWindows,