chore(plugin-coverage): improve nx helper, error handling, update docs

Author: Tlacenka

code-pushup.config.ts

@@ -61,6 +61,7 @@ const config: CoreConfig = {
           'unit-test',
           'integration-test',
           '--coverage',
+          '--skipNxCache',
         ],
       },
       reports: await getNxCoveragePaths(['unit-test', 'integration-test']),

packages/plugin-coverage/README.md

@@ -10,6 +10,9 @@ Measured coverage types are mapped to Code PushUp audits in the following way
 - the score is value converted to 0-1 range
 - missing coverage is mapped to issues in the audit details (uncalled functions, uncovered branches or lines)
 
+> [!IMPORTANT]
+> In order to successfully run your coverage tool and gather coverage results directly within the plugin, all your tests need to pass!
+
 ## Getting started
 
 1. If you haven't already, install [@code-pushup/cli](../cli/README.md) and create a configuration file.
@@ -75,11 +78,11 @@ Measured coverage types are mapped to Code PushUp audits in the following way
 
 Code coverage is a metric that indicates what percentage of source code is executed by unit tests. It can give insights into test effectiveness and uncover parts of source code that would otherwise go untested.
 
-- Statement coverage: Measures how many statements are executed in at least one test.
-- Line coverage: Measures how many lines are executed in at least one test. Unlike statement coverage, any partially executed line counts towards line coverage.
-- Condition coverage: Measures all condition values (`true`/`false`) evaluated for a conditional statement in at least one test.
-- Branch coverage: Measures how many branches are executed as a result of conditional statements (`if`/`else` and other) in at least one test. In case of short-circuit logic, only executed paths are counted in. Unlike condition coverage, it does not ensure all combinations of condition values are tested.
-- Function coverage: Measures how many functions are called in at least one test. Argument values, usage of optional arguments or default values is irrelevant for this metric.
+- **Statement coverage**: Measures how many statements are executed in at least one test.
+- **Line coverage**: Measures how many lines are executed in at least one test. Unlike statement coverage, any partially executed line counts towards line coverage.
+- **Condition coverage**: Measures all condition values (`true`/`false`) evaluated for a conditional statement in at least one test.
+- **Branch coverage**: Measures how many branches are executed as a result of conditional statements (`if`/`else` and other) in at least one test. In case of short-circuit logic, only executed paths are counted in. Unlike condition coverage, it does not ensure all combinations of condition values are tested.
+- **Function coverage**: Measures how many functions are called in at least one test. Argument values, usage of optional arguments or default values is irrelevant for this metric.
 
 > [!IMPORTANT]
 > Please note that code coverage is not the same as test coverage. Test coverage measures the amount of acceptance criteria covered by tests and is hard to formally verify. This means that code coverage cannot guarantee that the designed software caters to the business requirements.
@@ -117,7 +120,7 @@ The plugin accepts the following parameters:
 
 - `coverageTypes`: An array of types of coverage that you wish to track. Supported values: `function`, `branch`, `line`. Defaults to all available types.
 - `reports`: Array of information about files with code coverage results - paths to results, path to project root the results belong to. LCOV format is supported for now.
-  - If you have an `nx` monorepo, you can adjust our helper function `getNxCoveragePaths` to get the path information automatically.
+  - If you have an Nx monorepo, you can adjust our helper function `getNxCoveragePaths` to get the path information automatically.
 - (optional) `coverageToolCommand`: If you wish to run your coverage tool to generate the results first, you may define it here.
 - (optional) `perfectScoreThreshold`: If your coverage goal is not 100%, you may define it here in range 0-1. Any score above the defined threshold will be given the perfect score. The value will stay unaffected.
 
@@ -204,3 +207,14 @@ For instance, the following can be an audit output for line coverage.
   }
 }
 ```
+
+### Providing coverage results in Nx monorepo
+
+As a part of the plugin, there is a `getNxCoveragePaths` helper for setting up paths to coverage results if you are using Nx. The helper accepts all relevant targets (e.g. `test` or `unit-test`) and searches for a coverage path option.
+Jest and Vitest configuration options are currently supported:
+
+- For `@nx/jest` executor it looks for the `coverageDirectory` option.
+- For `@nx/vite` executor it looks for the `reportsDirectory` option.
+
+> [!IMPORTANT]
+> Please note that you need to set up the coverage directory option in your `project.json` target options. Test configuration files are not searched.

packages/plugin-coverage/package.json

@@ -5,6 +5,7 @@
     "@code-pushup/models": "*",
     "@code-pushup/utils": "*",
     "parse-lcov": "^1.0.4",
+    "chalk": "^5.3.0",
     "zod": "^3.22.4"
   },
   "peerDependencies": {

packages/plugin-coverage/src/lib/nx/coverage-paths.ts

@@ -1,16 +1,18 @@
-import type { ProjectGraphProjectNode } from '@nx/devkit';
+import type { ProjectGraphProjectNode, TargetConfiguration } from '@nx/devkit';
+import chalk from 'chalk';
 import { join } from 'node:path';
 import { CoverageResult } from '../config';
 
 /**
- *
- * @param coverageFolder root folder containing all coverage results
  * @param targets nx targets to be used for measuring coverage, test by default
  * @returns An array of coverage result information for the coverage plugin.
  */
 export async function getNxCoveragePaths(
   targets: string[] = ['test'],
 ): Promise<CoverageResult[]> {
+  console.info(
+    chalk.bold('💡 Gathering coverage from the following nx projects:'),
+  );
   const { createProjectGraphAsync } = await import('@nx/devkit');
   const { nodes } = await createProjectGraphAsync({ exitOnError: false });
 
@@ -19,23 +21,21 @@ export async function getNxCoveragePaths(
       hasNxTarget(graph, target),
     );
 
-    return relevantNodes
-      .map(node => node.data)
-      .map<CoverageResult>(projectConfig => {
-        const { reportsDirectory } = projectConfig.targets?.[target]
-          ?.options as {
-          reportsDirectory: string;
-        };
-
-        const rootToReportsDir = join(projectConfig.root, reportsDirectory);
-
-        return {
-          pathToProject: projectConfig.root,
-          resultsPath: join(rootToReportsDir, 'lcov.info'),
-        };
-      });
+    return relevantNodes.map<CoverageResult>(({ name, data }) => {
+      const targetConfig = data.targets?.[target] as TargetConfiguration;
+      const coveragePath = getCoveragePathForTarget(target, targetConfig, name);
+      const rootToReportsDir = join(data.root, coveragePath);
+
+      console.info(`- ${name}: ${target}`);
+
+      return {
+        pathToProject: data.root,
+        resultsPath: join(rootToReportsDir, 'lcov.info'),
+      };
+    });
   });
 
+  console.info('\n');
   return coverageResults.flat();
 }
 
@@ -45,3 +45,40 @@ function hasNxTarget(
 ): boolean {
   return project.data.targets != null && target in project.data.targets;
 }
+
+function getCoveragePathForTarget(
+  target: string,
+  targetConfig: TargetConfiguration,
+  projectName: string,
+): string {
+  if (targetConfig.executor?.includes('@nx/vite')) {
+    const { reportsDirectory } = targetConfig.options as {
+      reportsDirectory?: string;
+    };
+
+    if (reportsDirectory == null) {
+      throw new Error(
+        `Coverage configuration not found for target ${target} in ${projectName}. Define your Vitest coverage directory in the reportsDirectory option.`,
+      );
+    }
+
+    return reportsDirectory;
+  }
+
+  if (targetConfig.executor?.includes('@nx/jest')) {
+    const { coverageDirectory } = targetConfig.options as {
+      coverageDirectory?: string;
+    };
+
+    if (coverageDirectory == null) {
+      throw new Error(
+        `Coverage configuration not found for target ${target} in ${projectName}. Define your Jest coverage directory in the coverageDirectory option.`,
+      );
+    }
+    return coverageDirectory;
+  }
+
+  throw new Error(
+    `Unsupported executor ${targetConfig.executor}. @nx/vite and @nx/jest are currently supported.`,
+  );
+}

packages/plugin-coverage/src/lib/runner/index.ts

@@ -1,7 +1,9 @@
+import chalk from 'chalk';
 import { writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { AuditOutputs, RunnerConfig } from '@code-pushup/models';
+import type { AuditOutputs, RunnerConfig } from '@code-pushup/models';
 import {
+  ProcessError,
   ensureDirectoryExists,
   executeProcess,
   readJsonFile,
@@ -18,7 +20,21 @@ export async function executeRunner(): Promise<void> {
   // Run coverage tool if provided
   if (coverageToolCommand != null) {
     const { command, args } = coverageToolCommand;
-    await executeProcess({ command, args });
+
+    try {
+      await executeProcess({ command, args });
+    } catch (error) {
+      if (error instanceof ProcessError) {
+        console.info(chalk.bold('stdout from failed process:'));
+        console.info(error.stdout);
+        console.error(chalk.bold('stderr from failed process:'));
+        console.error(error.stderr);
+      }
+
+      throw new Error(
+        'Coverage plugin: Running coverage tool failed. Make sure all your provided tests are passing.',
+      );
+    }
   }
 
   // Caculate coverage from LCOV results

packages/plugin-coverage/src/lib/runner/runner.integration.test.ts

@@ -44,7 +44,7 @@ describe('createRunnerConfig', () => {
 
 describe('executeRunner', () => {
   it('should successfully execute runner', async () => {
-    const config = {
+    const config: FinalCoveragePluginConfig = {
       reports: [
         {
           resultsPath: join(