Add ignoreStderr option to ignore stderr output in test results (#101)

When enabled, stderr is not automatically treated as test failure.
Tests are marked as failed only if actual failure markers are detected.

Author: viordash

README.md

@@ -53,6 +53,16 @@ Enable logging for debugging:
 - logpanel: Shows logs in VS Code's Output panel ("CppUTest Test Adapter Log").
 - logfile: Saves logs to a file. Use absolute paths and ensure the directory exists.
 
+## Ignore Stderr
+By default, any output to stderr is treated as a test failure. If your tests produce warnings or logs on stderr without actually failing, you can disable this behavior:
+
+```json
+{
+  "cpputestTestAdapter.ignoreStderr": true
+}
+```
+- When `true`, stderr output is not automatically treated as test failure. Tests are marked as failed only if actual failure markers are detected in the output.
+
 ## Debugging
 To debug your tests, configure your launch.json like you would with any debugger (in this case gdb):
 

package.json

@@ -132,6 +132,12 @@
           "default": "objdump",
           "type": "string",
           "scope": "resource"
+        },
+        "cpputestTestAdapter.ignoreStderr": {
+          "description": "Ignore stderr output when determining test result. When true, stderr is not treated as test failure",
+          "default": false,
+          "type": "boolean",
+          "scope": "resource"
         }
       }
     }

src/Infrastructure/ExecutableRunner.ts

@@ -22,6 +22,7 @@ export class RunResult {
 export interface ExecutableRunnerOptions {
   workingDirectory?: string;
   objDumpExecutable?: string;
+  ignoreStderr?: boolean;
 }
 
 export default class ExecutableRunner {
@@ -31,6 +32,7 @@ export default class ExecutableRunner {
   private readonly command: string;
   private readonly workingDirectory: string;
   private readonly objDumpExecutable: string;
+  private readonly ignoreStderr: boolean;
   private readonly tempFile: string;
   public readonly Name: string;
   private dumpCached: boolean;
@@ -49,6 +51,7 @@ export default class ExecutableRunner {
     this.workingDirectory = this.isEmptyString(wd) ? dirname(command) : wd!.trim();
 
     this.objDumpExecutable = options?.objDumpExecutable ?? "objdump";
+    this.ignoreStderr = options?.ignoreStderr ?? false;
     this.Name = basename(command);
     this.tempFile = `${this.Name}.dump`
     this.dumpCached = false;
@@ -148,7 +151,7 @@ export default class ExecutableRunner {
           console.error('stderr', error);
           reject(stderr);
         }
-        if (stderr.trim() != "") {
+        if (!this.ignoreStderr && stderr.trim() != "") {
           resolve(new RunResult(RunResultStatus.Error, stderr));
         } else if (error && error.code !== 0) {
           resolve(new RunResult(RunResultStatus.Failure, stdout));

src/Infrastructure/IWorkspaceConfiguration.ts

@@ -8,4 +8,5 @@ export interface IWorkspaceConfiguration {
   debugLaunchConfigurationName: string | undefined;
   objDumpExecutable: string | undefined;
   preLaunchTask: string;
+  ignoreStderr: boolean;
 }

src/Infrastructure/SettingsProvider.ts

@@ -21,7 +21,8 @@ export abstract class SettingsProvider {
   public abstract GetTestPath(): string;
   public abstract GetPreLaunchTask(): string;
   public abstract GetDebugConfiguration(): (IDebugConfiguration | undefined);
-  public abstract GetWorkspaceFolders(): readonly IWorkspaceFolder[] | undefined
+  public abstract GetWorkspaceFolders(): readonly IWorkspaceFolder[] | undefined;
+  public abstract GetIgnoreStderr(): boolean;
   protected abstract GetCurrentFilename(): string
   protected abstract GetCurrentWorkspaceFolder(): string
   protected abstract GlobFiles(wildcardString: string): string[]

src/Infrastructure/VscodeSettingsProvider.ts

@@ -24,7 +24,8 @@ export default class VscodeSettingsProvider extends SettingsProvider {
           testExecutable: wsConfig["testExecutable"],
           testExecutablePath: wsConfig["testExecutablePath"],
           testLocationFetchMode: wsConfig["testLocationFetchMode"],
-          preLaunchTask: wsConfig["preLaunchTask"]
+          preLaunchTask: wsConfig["preLaunchTask"],
+          ignoreStderr: wsConfig["ignoreStderr"]
         }
       }
     })
@@ -54,6 +55,10 @@ export default class VscodeSettingsProvider extends SettingsProvider {
     return this.ResolveSettingsVariable(this.config.testExecutablePath);
   }
 
+  public GetIgnoreStderr(): boolean {
+    return this.config.ignoreStderr ?? false;
+  }
+
   public GetDebugConfiguration(): (IDebugConfiguration | undefined) {
     // Thanks to: https://github.com/matepek/vscode-catch2-test-adapter/blob/9a2e9f5880ef3907d80ff99f3d6d028270923c95/src/Configurations.ts#L125
     if (vscode.workspace.workspaceFolders === undefined) {

src/adapter.ts

@@ -147,7 +147,8 @@ export class CppUTestAdapter implements TestAdapter {
 	private GetExecutionOptions(): ExecutableRunnerOptions | undefined {
 		return {
 			objDumpExecutable: this.settingsProvider.GetObjDumpPath(),
-			workingDirectory: this.settingsProvider.GetTestPath()
+			workingDirectory: this.settingsProvider.GetTestPath(),
+			ignoreStderr: this.settingsProvider.GetIgnoreStderr()
 		}
 	}
 

tests/ExecutableRunner.spec.ts

@@ -228,6 +228,58 @@ describe("ExecutableRunner should", () => {
       expect(capturedOptions.cwd).to.equal(customWorkingDir);
     });
   });
+
+  describe("ignoreStderr option", () => {
+    it("should return Error status when stderr is present and ignoreStderr is false", async () => {
+      const log = mock<Log>();
+      const stderrOutput = "some stderr warning";
+      const processExecuter = setupMockCalls(undefined, "stdout output", stderrOutput);
+      const command = "runnable";
+
+      const runner = new ExecutableRunner(processExecuter, command, log, { ignoreStderr: false });
+      const result = await runner.RunTest("myGroup", "myTest");
+
+      expect(result).to.be.deep.equal(new RunResult(RunResultStatus.Error, stderrOutput));
+    });
+
+    it("should return Error status when stderr is present and ignoreStderr is not specified", async () => {
+      const log = mock<Log>();
+      const stderrOutput = "some stderr warning";
+      const processExecuter = setupMockCalls(undefined, "stdout output", stderrOutput);
+      const command = "runnable";
+
+      const runner = new ExecutableRunner(processExecuter, command, log);
+      const result = await runner.RunTest("myGroup", "myTest");
+
+      expect(result).to.be.deep.equal(new RunResult(RunResultStatus.Error, stderrOutput));
+    });
+
+    it("should ignore stderr and return Success when ignoreStderr is true and test passes", async () => {
+      const log = mock<Log>();
+      const stdoutOutput = "TEST(myGroup, myTest) - 0 ms";
+      const stderrOutput = "some stderr warning";
+      const processExecuter = setupMockCalls(undefined, stdoutOutput, stderrOutput);
+      const command = "runnable";
+
+      const runner = new ExecutableRunner(processExecuter, command, log, { ignoreStderr: true });
+      const result = await runner.RunTest("myGroup", "myTest");
+
+      expect(result).to.be.deep.equal(new RunResult(RunResultStatus.Success, stdoutOutput));
+    });
+
+    it("should ignore stderr and return Failure when ignoreStderr is true and test fails", async () => {
+      const log = mock<Log>();
+      const stdoutOutput = "TEST(myGroup, myTest) failed";
+      const stderrOutput = "some stderr warning";
+      const processExecuter = setupMockCalls(new ExecError(1), stdoutOutput, stderrOutput);
+      const command = "runnable";
+
+      const runner = new ExecutableRunner(processExecuter, command, log, { ignoreStderr: true });
+      const result = await runner.RunTest("myGroup", "myTest");
+
+      expect(result).to.be.deep.equal(new RunResult(RunResultStatus.Failure, stdoutOutput));
+    });
+  });
 })
 
 function setupMockCalls(error: ExecException | undefined, returnValue: string, errorValue: string) {

tests/RegexResultParser.spec.ts

@@ -43,4 +43,17 @@ describe("RegexResultParser should", () => {
       expect(resultParser.GetResult(new RunResult(statusFromRunner, stringFromRunner))).to.be.deep.eq(expectedTestResult);
     })
   })
+
+  describe("ParseResultError", () => {
+    it("should treat stderr as failure", () => {
+      const resultParser = new RegexResultParser();
+      const stderrOutput = "Some warning output to stderr";
+      const runResult = new RunResult(RunResultStatus.Error, stderrOutput);
+
+      const result = resultParser.GetResult(runResult);
+
+      expect(result.state).to.equal(TestState.Failed);
+      expect(result.message).to.equal(stderrOutput);
+    });
+  });
 });

tests/SettingsProvider.spec.ts

@@ -47,6 +47,9 @@ class TestSettingsProvider extends SettingsProvider {
             uri: "."
         }];
     }
+    GetIgnoreStderr(): boolean {
+        return this.config.ignoreStderr ?? false;
+    }
     protected GetCurrentFilename(): string {
         return "myFile.c";
     }
@@ -74,7 +77,8 @@ describe("SettingsProvider should", () => {
         testExecutable: "",
         testExecutablePath: "",
         testLocationFetchMode: "",
-        preLaunchTask: ""
+        preLaunchTask: "",
+        ignoreStderr: false
     };
     let filesToFind = [""];