fix(observability,session-status): don't mark suppressNotFoundErrors commands as test failures

nightwatch/globals.js

@@ -307,12 +307,22 @@ module.exports = {
           try {
             const testName = test?.testcase;
             const eventData = (testName && test?.envelope?.[testName]?.testcase) || null;
+            // Skip commands the caller opted out of via suppressNotFoundErrors:true
+            // and trust Nightwatch's envelope-level rollup over a stray command-level
+            // fail status. Same defect shape as testObservability.js sendTestRunEvent.
+            // When eventData itself is missing (session aborted before TestRunStarted
+            // populated the envelope), failedCommand stays null and status defaults
+            // to 'passed' — matches the pre-fix behaviour for that edge case.
             const failedCommand = eventData?.commands && Array.isArray(eventData.commands)
-              ? eventData.commands.find(cmd => cmd.status === 'fail')
+              ? eventData.commands.find(cmd => cmd.status === 'fail' && !helper.isSuppressedFailure(cmd))
               : null;
-            const status = failedCommand ? 'failed' : 'passed';
+            const envelopePassed = !!eventData
+              && (eventData.status === 'pass')
+              && ((eventData.failed || 0) === 0)
+              && ((eventData.errors || 0) === 0);
+            const status = (failedCommand && !envelopePassed) ? 'failed' : 'passed';
             let reason = '';
-            if (failedCommand && failedCommand.result) {
+            if (status === 'failed' && failedCommand && failedCommand.result) {
               reason = (failedCommand.result.message || failedCommand.result.stack || 'Test failed').toString().slice(0, 280);
             }
             const payload = JSON.stringify({status, reason});

src/testObservability.js

@@ -537,8 +537,12 @@ class TestObservability {
       testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(startTimestamp).toISOString();
       testData.result = 'passed';
       if (eventData && eventData.commands && Array.isArray(eventData.commands)) {
-        const failedCommand = eventData.commands.find(cmd => cmd.status === 'fail');
-        if (failedCommand) {
+        const failedCommand = eventData.commands.find(cmd => cmd.status === 'fail' && !helper.isSuppressedFailure(cmd));
+        // Envelope-level rollup: when Nightwatch itself reports the testcase
+        // as passed (no failed assertions / errors), trust the rollup over a
+        // stray command-level fail status that did not propagate.
+        const envelopePassed = (eventData.status === 'pass') && ((eventData.failed || 0) === 0) && ((eventData.errors || 0) === 0);
+        if (failedCommand && !envelopePassed) {
           testData.result = 'failed';
           if (failedCommand.result) {
             testData.failure = [

src/utils/helper.js

@@ -62,6 +62,30 @@ exports.isUndefined = value => (value === undefined || value === null);
 
 exports.isObject = value => (!this.isUndefined(value) && value.constructor === Object);
 
+// A Nightwatch command is recorded with status:'fail' even when the caller
+// explicitly opted into "not found is fine" via `suppressNotFoundErrors:true`.
+// Those commands carry no failure semantics for the test and must not flip
+// test/session status. Only `args[0]` is inspected — Nightwatch's reporter
+// always serializes element-command options as the first positional argument,
+// and either as an object literal or as a JSON-encoded string depending on
+// the reporter path. Both shapes must be accepted.
+exports.isSuppressedFailure = (cmd) => {
+  if (!cmd || cmd.status !== 'fail' || !Array.isArray(cmd.args) || cmd.args.length === 0) {return false}
+  const first = cmd.args[0];
+  let opts = first;
+  if (typeof first === 'string') {
+    try {
+      opts = JSON.parse(first);
+    } catch (e) {
+      Logger.debug(`isSuppressedFailure: could not parse args[0] for cmd ${cmd.name || '<unnamed>'}: ${e.message}`);
+
+      return false;
+    }
+  }
+
+  return !!(opts && typeof opts === 'object' && opts.suppressNotFoundErrors === true);
+};
+
 exports.isTestObservabilitySession = () => {
   return process.env.BROWSERSTACK_TEST_OBSERVABILITY === 'true' || 
          process.env.BROWSERSTACK_TEST_REPORTING === 'true';

test/src/test-observability/sendTestRunEvent.js

@@ -0,0 +1,164 @@
+const assert = require('assert');
+const sinon = require('sinon');
+
+const helper = require('../../../src/utils/helper');
+const TestMap = require('../../../src/utils/testMap');
+const TestObservability = require('../../../src/testObservability');
+
+// Regression coverage for SDK-5914 / suppressNotFoundErrors.
+//
+// Before this fix, sendTestRunEvent walked eventData.commands for any
+// status:'fail' record and flipped testData.result to 'failed' even when
+// the failing command was a Nightwatch `isVisible({suppressNotFoundErrors:true})`
+// lookup whose absence is an expected, callsite-opted-into outcome.
+// The bug shipped because this function had no unit coverage at all.
+describe('TestObservability - sendTestRunEvent (suppressNotFoundErrors)', function () {
+  const buildTest = (commands, envelopeRollup = {status: 'pass', failed: 0, errors: 0}) => ({
+    metadata: {
+      name: 'Conditional Suite',
+      tags: [],
+      modulePath: '/tmp/observabilityBugRepro.js',
+      host: 'hub-cloud.browserstack.com',
+      sessionId: 'session-id-stub',
+      sessionCapabilities: {}
+    },
+    testcase: 'conditional test',
+    testCaseData: () => '',
+    settings: {desiredCapabilities: {'bstack:options': {osVersion: '11'}}},
+    envelope: {
+      'conditional test': {
+        startTimestamp: 1700000000000,
+        testcase: {
+          endTimestamp: 1700000001000,
+          commands,
+          ...envelopeRollup
+        }
+      }
+    }
+  });
+
+  beforeEach(() => {
+    this.sandbox = sinon.createSandbox();
+    this.testObservability = new TestObservability();
+
+    this.sandbox.stub(this.testObservability, 'getTestBody').returns('');
+    this.sandbox.stub(this.testObservability, 'processTestRunData').resolves();
+    this.sandbox.stub(helper, 'getCloudProvider').returns('automate');
+    this.sandbox.stub(helper, 'getIntegrationsObject').returns({});
+    this.sandbox.stub(helper, 'isTestObservabilitySession').returns(true);
+    this.sandbox.stub(helper, 'isAccessibilitySession').returns(false);
+    this.sandbox.stub(TestMap, 'getSessionSnapshot').returns(null);
+
+    this.uploaded = null;
+    this.uploadStub = this.sandbox.stub(helper, 'uploadEventData').callsFake(async (payload) => {
+      this.uploaded = payload;
+    });
+  });
+
+  afterEach(() => {
+    this.sandbox.restore();
+  });
+
+  it('marks the test passed when the only failing command opted into suppressNotFoundErrors (args as object)', async () => {
+    const commands = [
+      {name: 'url', args: ['https://www.google.com'], status: 'pass'},
+      {
+        name: 'isVisible',
+        args: [{selector: '#may-or-may-not-exist', suppressNotFoundErrors: true, timeout: 2000}, null],
+        status: 'fail',
+        result: {message: 'Element not found', stack: '', name: 'Error'}
+      }
+    ];
+
+    await this.testObservability.sendTestRunEvent('TestRunFinished', buildTest(commands), 'uuid-1');
+
+    sinon.assert.calledOnce(this.uploadStub);
+    assert.strictEqual(this.uploaded.event_type, 'TestRunFinished');
+    assert.strictEqual(this.uploaded.test_run.result, 'passed');
+    assert.ok(!('failure' in this.uploaded.test_run), 'expected no failure field on passed test');
+    assert.ok(!('failure_reason' in this.uploaded.test_run), 'expected no failure_reason field on passed test');
+  });
+
+  it('marks the test passed when args[0] is a JSON-encoded string carrying suppressNotFoundErrors', async () => {
+    // Some Nightwatch reporter paths serialize the options object to a JSON
+    // string in command.args[0] — the customer's CHROME_148__observabilityBugRepro.json
+    // is the canonical example. The fix must handle both shapes.
+    const commands = [
+      {
+        name: 'isVisible',
+        args: ['{"selector":"#may-or-may-not-exist","suppressNotFoundErrors":true,"timeout":2000}', null],
+        status: 'fail',
+        result: {message: 'Element not found', stack: ''}
+      }
+    ];
+
+    await this.testObservability.sendTestRunEvent('TestRunFinished', buildTest(commands), 'uuid-2');
+
+    assert.strictEqual(this.uploaded.test_run.result, 'passed');
+  });
+
+  it('still marks the test failed when a real assertion failure is present', async () => {
+    // Envelope rollup says failed:1 — a real failure happened. The fix must
+    // NOT suppress that. This is the contrast case that prevents the patch
+    // from silently downgrading every failing test to passed.
+    const commands = [
+      {
+        name: 'assert.titleContains',
+        args: ['Google'],
+        status: 'fail',
+        result: {message: 'Expected title to contain "Google"', stack: 'AssertionError', name: 'AssertionError'}
+      }
+    ];
+
+    await this.testObservability.sendTestRunEvent(
+      'TestRunFinished',
+      buildTest(commands, {status: 'fail', failed: 1, errors: 0}),
+      'uuid-3'
+    );
+
+    assert.strictEqual(this.uploaded.test_run.result, 'failed');
+    assert.strictEqual(this.uploaded.test_run.failure_type, 'AssertionError');
+    // Lock the wire-shape that the original bug malformed (failure_reason:null,
+    // backtrace:["",""]). The patch must propagate the real failure detail.
+    assert.strictEqual(this.uploaded.test_run.failure_reason, 'Expected title to contain "Google"');
+    assert.deepStrictEqual(
+      this.uploaded.test_run.failure[0].backtrace,
+      ['Expected title to contain "Google"', 'AssertionError']
+    );
+  });
+
+  it('still marks the test failed when a non-suppressed command failed alongside a suppressed one', async () => {
+    // Mixed case: one suppressed isVisible + one real failure. Envelope rollup
+    // disagrees with "all passed", so we must propagate the real failure.
+    const commands = [
+      {
+        name: 'isVisible',
+        args: [{selector: '#optional', suppressNotFoundErrors: true}, null],
+        status: 'fail',
+        result: {message: 'Element not found'}
+      },
+      {
+        name: 'click',
+        args: ['#mandatory'],
+        status: 'fail',
+        result: {message: 'Element #mandatory not found', stack: 'NoSuchElementError', name: 'NoSuchElementError'}
+      }
+    ];
+
+    await this.testObservability.sendTestRunEvent(
+      'TestRunFinished',
+      buildTest(commands, {status: 'fail', failed: 0, errors: 1}),
+      'uuid-4'
+    );
+
+    assert.strictEqual(this.uploaded.test_run.result, 'failed');
+    // failedCommand must skip the suppressed one and pick the real one — confirm
+    // by asserting the failure_reason references the mandatory selector, not the
+    // suppressed optional one.
+    assert.strictEqual(this.uploaded.test_run.failure_type, 'UnhandledError');
+    assert.ok(
+      this.uploaded.test_run.failure_reason.includes('#mandatory'),
+      `expected failure_reason to reference the real failure, got: ${this.uploaded.test_run.failure_reason}`
+    );
+  });
+});

test/src/utils/helper.js

@@ -353,3 +353,69 @@ describe('isBrowserstackInfra', () => {
   });
 
 });
+
+describe('isSuppressedFailure', () => {
+  // Shared primitive used by both src/testObservability.js (sendTestRunEvent)
+  // and nightwatch/globals.js (setSessionStatus). Covers both call sites by
+  // reference — any regression here breaks both surfaces equivalently.
+  let isSuppressedFailure;
+  before(() => {
+    isSuppressedFailure = require('../../../src/utils/helper').isSuppressedFailure;
+  });
+
+  it('returns false for null / undefined commands', () => {
+    expect(isSuppressedFailure(null)).to.be.false;
+    expect(isSuppressedFailure(undefined)).to.be.false;
+  });
+
+  it('returns false for commands that did not fail', () => {
+    expect(isSuppressedFailure({status: 'pass', args: [{suppressNotFoundErrors: true}]})).to.be.false;
+  });
+
+  it('returns false when args is missing or empty', () => {
+    expect(isSuppressedFailure({status: 'fail'})).to.be.false;
+    expect(isSuppressedFailure({status: 'fail', args: []})).to.be.false;
+  });
+
+  it('returns false when args is not an array', () => {
+    expect(isSuppressedFailure({status: 'fail', args: {suppressNotFoundErrors: true}})).to.be.false;
+    expect(isSuppressedFailure({status: 'fail', args: 'not-an-array'})).to.be.false;
+  });
+
+  it('returns false when args[0] is a primitive', () => {
+    expect(isSuppressedFailure({status: 'fail', args: [null]})).to.be.false;
+    expect(isSuppressedFailure({status: 'fail', args: [42]})).to.be.false;
+    expect(isSuppressedFailure({status: 'fail', args: [true]})).to.be.false;
+  });
+
+  it('returns true when args[0] is an object with suppressNotFoundErrors:true', () => {
+    expect(isSuppressedFailure({
+      status: 'fail',
+      args: [{selector: '#x', suppressNotFoundErrors: true, timeout: 2000}, null]
+    })).to.be.true;
+  });
+
+  it('returns true when args[0] is a JSON string carrying suppressNotFoundErrors:true', () => {
+    expect(isSuppressedFailure({
+      status: 'fail',
+      args: ['{"selector":"#x","suppressNotFoundErrors":true,"timeout":2000}', null]
+    })).to.be.true;
+  });
+
+  it('returns false when args[0] is a malformed JSON string (safe default)', () => {
+    // Unparseable strings default to "not suppressed" — i.e., the test stays
+    // failed. That's the safer direction, matches the rest of the guard's
+    // bias to under-suppress rather than over-suppress.
+    expect(isSuppressedFailure({status: 'fail', args: ['{this is not json']})).to.be.false;
+  });
+
+  it('returns false when args[0] is an object without suppressNotFoundErrors', () => {
+    expect(isSuppressedFailure({status: 'fail', args: [{selector: '#x', timeout: 2000}]})).to.be.false;
+  });
+
+  it('returns false when suppressNotFoundErrors is falsy', () => {
+    expect(isSuppressedFailure({status: 'fail', args: [{suppressNotFoundErrors: false}]})).to.be.false;
+    expect(isSuppressedFailure({status: 'fail', args: [{suppressNotFoundErrors: 'true'}]})).to.be.false;
+  });
+
+});