diff --git a/nightwatch/globals.js b/nightwatch/globals.js index f6852da..16f029b 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -307,12 +307,22 @@ module.exports = { try { const testName = test?.testcase; const eventData = (testName && test?.envelope?.[testName]?.testcase) || null; + // Skip commands the caller opted out of via suppressNotFoundErrors:true + // and trust Nightwatch's envelope-level rollup over a stray command-level + // fail status. Same defect shape as testObservability.js sendTestRunEvent. + // When eventData itself is missing (session aborted before TestRunStarted + // populated the envelope), failedCommand stays null and status defaults + // to 'passed' — matches the pre-fix behaviour for that edge case. const failedCommand = eventData?.commands && Array.isArray(eventData.commands) - ? eventData.commands.find(cmd => cmd.status === 'fail') + ? eventData.commands.find(cmd => cmd.status === 'fail' && !helper.isSuppressedFailure(cmd)) : null; - const status = failedCommand ? 'failed' : 'passed'; + const envelopePassed = !!eventData + && (eventData.status === 'pass') + && ((eventData.failed || 0) === 0) + && ((eventData.errors || 0) === 0); + const status = (failedCommand && !envelopePassed) ? 'failed' : 'passed'; let reason = ''; - if (failedCommand && failedCommand.result) { + if (status === 'failed' && failedCommand && failedCommand.result) { reason = (failedCommand.result.message || failedCommand.result.stack || 'Test failed').toString().slice(0, 280); } const payload = JSON.stringify({status, reason}); diff --git a/src/testObservability.js b/src/testObservability.js index dcd336e..0720ec1 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -537,8 +537,12 @@ class TestObservability { testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(startTimestamp).toISOString(); testData.result = 'passed'; if (eventData && eventData.commands && Array.isArray(eventData.commands)) { - const failedCommand = eventData.commands.find(cmd => cmd.status === 'fail'); - if (failedCommand) { + const failedCommand = eventData.commands.find(cmd => cmd.status === 'fail' && !helper.isSuppressedFailure(cmd)); + // Envelope-level rollup: when Nightwatch itself reports the testcase + // as passed (no failed assertions / errors), trust the rollup over a + // stray command-level fail status that did not propagate. + const envelopePassed = (eventData.status === 'pass') && ((eventData.failed || 0) === 0) && ((eventData.errors || 0) === 0); + if (failedCommand && !envelopePassed) { testData.result = 'failed'; if (failedCommand.result) { testData.failure = [ diff --git a/src/utils/helper.js b/src/utils/helper.js index ffae9a5..eb464a1 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -62,6 +62,30 @@ exports.isUndefined = value => (value === undefined || value === null); exports.isObject = value => (!this.isUndefined(value) && value.constructor === Object); +// A Nightwatch command is recorded with status:'fail' even when the caller +// explicitly opted into "not found is fine" via `suppressNotFoundErrors:true`. +// Those commands carry no failure semantics for the test and must not flip +// test/session status. Only `args[0]` is inspected — Nightwatch's reporter +// always serializes element-command options as the first positional argument, +// and either as an object literal or as a JSON-encoded string depending on +// the reporter path. Both shapes must be accepted. +exports.isSuppressedFailure = (cmd) => { + if (!cmd || cmd.status !== 'fail' || !Array.isArray(cmd.args) || cmd.args.length === 0) {return false} + const first = cmd.args[0]; + let opts = first; + if (typeof first === 'string') { + try { + opts = JSON.parse(first); + } catch (e) { + Logger.debug(`isSuppressedFailure: could not parse args[0] for cmd ${cmd.name || ''}: ${e.message}`); + + return false; + } + } + + return !!(opts && typeof opts === 'object' && opts.suppressNotFoundErrors === true); +}; + exports.isTestObservabilitySession = () => { return process.env.BROWSERSTACK_TEST_OBSERVABILITY === 'true' || process.env.BROWSERSTACK_TEST_REPORTING === 'true'; diff --git a/test/src/test-observability/sendTestRunEvent.js b/test/src/test-observability/sendTestRunEvent.js new file mode 100644 index 0000000..8271507 --- /dev/null +++ b/test/src/test-observability/sendTestRunEvent.js @@ -0,0 +1,164 @@ +const assert = require('assert'); +const sinon = require('sinon'); + +const helper = require('../../../src/utils/helper'); +const TestMap = require('../../../src/utils/testMap'); +const TestObservability = require('../../../src/testObservability'); + +// Regression coverage for SDK-5914 / suppressNotFoundErrors. +// +// Before this fix, sendTestRunEvent walked eventData.commands for any +// status:'fail' record and flipped testData.result to 'failed' even when +// the failing command was a Nightwatch `isVisible({suppressNotFoundErrors:true})` +// lookup whose absence is an expected, callsite-opted-into outcome. +// The bug shipped because this function had no unit coverage at all. +describe('TestObservability - sendTestRunEvent (suppressNotFoundErrors)', function () { + const buildTest = (commands, envelopeRollup = {status: 'pass', failed: 0, errors: 0}) => ({ + metadata: { + name: 'Conditional Suite', + tags: [], + modulePath: '/tmp/observabilityBugRepro.js', + host: 'hub-cloud.browserstack.com', + sessionId: 'session-id-stub', + sessionCapabilities: {} + }, + testcase: 'conditional test', + testCaseData: () => '', + settings: {desiredCapabilities: {'bstack:options': {osVersion: '11'}}}, + envelope: { + 'conditional test': { + startTimestamp: 1700000000000, + testcase: { + endTimestamp: 1700000001000, + commands, + ...envelopeRollup + } + } + } + }); + + beforeEach(() => { + this.sandbox = sinon.createSandbox(); + this.testObservability = new TestObservability(); + + this.sandbox.stub(this.testObservability, 'getTestBody').returns(''); + this.sandbox.stub(this.testObservability, 'processTestRunData').resolves(); + this.sandbox.stub(helper, 'getCloudProvider').returns('automate'); + this.sandbox.stub(helper, 'getIntegrationsObject').returns({}); + this.sandbox.stub(helper, 'isTestObservabilitySession').returns(true); + this.sandbox.stub(helper, 'isAccessibilitySession').returns(false); + this.sandbox.stub(TestMap, 'getSessionSnapshot').returns(null); + + this.uploaded = null; + this.uploadStub = this.sandbox.stub(helper, 'uploadEventData').callsFake(async (payload) => { + this.uploaded = payload; + }); + }); + + afterEach(() => { + this.sandbox.restore(); + }); + + it('marks the test passed when the only failing command opted into suppressNotFoundErrors (args as object)', async () => { + const commands = [ + {name: 'url', args: ['https://www.google.com'], status: 'pass'}, + { + name: 'isVisible', + args: [{selector: '#may-or-may-not-exist', suppressNotFoundErrors: true, timeout: 2000}, null], + status: 'fail', + result: {message: 'Element not found', stack: '', name: 'Error'} + } + ]; + + await this.testObservability.sendTestRunEvent('TestRunFinished', buildTest(commands), 'uuid-1'); + + sinon.assert.calledOnce(this.uploadStub); + assert.strictEqual(this.uploaded.event_type, 'TestRunFinished'); + assert.strictEqual(this.uploaded.test_run.result, 'passed'); + assert.ok(!('failure' in this.uploaded.test_run), 'expected no failure field on passed test'); + assert.ok(!('failure_reason' in this.uploaded.test_run), 'expected no failure_reason field on passed test'); + }); + + it('marks the test passed when args[0] is a JSON-encoded string carrying suppressNotFoundErrors', async () => { + // Some Nightwatch reporter paths serialize the options object to a JSON + // string in command.args[0] — the customer's CHROME_148__observabilityBugRepro.json + // is the canonical example. The fix must handle both shapes. + const commands = [ + { + name: 'isVisible', + args: ['{"selector":"#may-or-may-not-exist","suppressNotFoundErrors":true,"timeout":2000}', null], + status: 'fail', + result: {message: 'Element not found', stack: ''} + } + ]; + + await this.testObservability.sendTestRunEvent('TestRunFinished', buildTest(commands), 'uuid-2'); + + assert.strictEqual(this.uploaded.test_run.result, 'passed'); + }); + + it('still marks the test failed when a real assertion failure is present', async () => { + // Envelope rollup says failed:1 — a real failure happened. The fix must + // NOT suppress that. This is the contrast case that prevents the patch + // from silently downgrading every failing test to passed. + const commands = [ + { + name: 'assert.titleContains', + args: ['Google'], + status: 'fail', + result: {message: 'Expected title to contain "Google"', stack: 'AssertionError', name: 'AssertionError'} + } + ]; + + await this.testObservability.sendTestRunEvent( + 'TestRunFinished', + buildTest(commands, {status: 'fail', failed: 1, errors: 0}), + 'uuid-3' + ); + + assert.strictEqual(this.uploaded.test_run.result, 'failed'); + assert.strictEqual(this.uploaded.test_run.failure_type, 'AssertionError'); + // Lock the wire-shape that the original bug malformed (failure_reason:null, + // backtrace:["",""]). The patch must propagate the real failure detail. + assert.strictEqual(this.uploaded.test_run.failure_reason, 'Expected title to contain "Google"'); + assert.deepStrictEqual( + this.uploaded.test_run.failure[0].backtrace, + ['Expected title to contain "Google"', 'AssertionError'] + ); + }); + + it('still marks the test failed when a non-suppressed command failed alongside a suppressed one', async () => { + // Mixed case: one suppressed isVisible + one real failure. Envelope rollup + // disagrees with "all passed", so we must propagate the real failure. + const commands = [ + { + name: 'isVisible', + args: [{selector: '#optional', suppressNotFoundErrors: true}, null], + status: 'fail', + result: {message: 'Element not found'} + }, + { + name: 'click', + args: ['#mandatory'], + status: 'fail', + result: {message: 'Element #mandatory not found', stack: 'NoSuchElementError', name: 'NoSuchElementError'} + } + ]; + + await this.testObservability.sendTestRunEvent( + 'TestRunFinished', + buildTest(commands, {status: 'fail', failed: 0, errors: 1}), + 'uuid-4' + ); + + assert.strictEqual(this.uploaded.test_run.result, 'failed'); + // failedCommand must skip the suppressed one and pick the real one — confirm + // by asserting the failure_reason references the mandatory selector, not the + // suppressed optional one. + assert.strictEqual(this.uploaded.test_run.failure_type, 'UnhandledError'); + assert.ok( + this.uploaded.test_run.failure_reason.includes('#mandatory'), + `expected failure_reason to reference the real failure, got: ${this.uploaded.test_run.failure_reason}` + ); + }); +}); diff --git a/test/src/utils/helper.js b/test/src/utils/helper.js index 2d54cb7..b3a8dd3 100644 --- a/test/src/utils/helper.js +++ b/test/src/utils/helper.js @@ -353,3 +353,69 @@ describe('isBrowserstackInfra', () => { }); }); + +describe('isSuppressedFailure', () => { + // Shared primitive used by both src/testObservability.js (sendTestRunEvent) + // and nightwatch/globals.js (setSessionStatus). Covers both call sites by + // reference — any regression here breaks both surfaces equivalently. + let isSuppressedFailure; + before(() => { + isSuppressedFailure = require('../../../src/utils/helper').isSuppressedFailure; + }); + + it('returns false for null / undefined commands', () => { + expect(isSuppressedFailure(null)).to.be.false; + expect(isSuppressedFailure(undefined)).to.be.false; + }); + + it('returns false for commands that did not fail', () => { + expect(isSuppressedFailure({status: 'pass', args: [{suppressNotFoundErrors: true}]})).to.be.false; + }); + + it('returns false when args is missing or empty', () => { + expect(isSuppressedFailure({status: 'fail'})).to.be.false; + expect(isSuppressedFailure({status: 'fail', args: []})).to.be.false; + }); + + it('returns false when args is not an array', () => { + expect(isSuppressedFailure({status: 'fail', args: {suppressNotFoundErrors: true}})).to.be.false; + expect(isSuppressedFailure({status: 'fail', args: 'not-an-array'})).to.be.false; + }); + + it('returns false when args[0] is a primitive', () => { + expect(isSuppressedFailure({status: 'fail', args: [null]})).to.be.false; + expect(isSuppressedFailure({status: 'fail', args: [42]})).to.be.false; + expect(isSuppressedFailure({status: 'fail', args: [true]})).to.be.false; + }); + + it('returns true when args[0] is an object with suppressNotFoundErrors:true', () => { + expect(isSuppressedFailure({ + status: 'fail', + args: [{selector: '#x', suppressNotFoundErrors: true, timeout: 2000}, null] + })).to.be.true; + }); + + it('returns true when args[0] is a JSON string carrying suppressNotFoundErrors:true', () => { + expect(isSuppressedFailure({ + status: 'fail', + args: ['{"selector":"#x","suppressNotFoundErrors":true,"timeout":2000}', null] + })).to.be.true; + }); + + it('returns false when args[0] is a malformed JSON string (safe default)', () => { + // Unparseable strings default to "not suppressed" — i.e., the test stays + // failed. That's the safer direction, matches the rest of the guard's + // bias to under-suppress rather than over-suppress. + expect(isSuppressedFailure({status: 'fail', args: ['{this is not json']})).to.be.false; + }); + + it('returns false when args[0] is an object without suppressNotFoundErrors', () => { + expect(isSuppressedFailure({status: 'fail', args: [{selector: '#x', timeout: 2000}]})).to.be.false; + }); + + it('returns false when suppressNotFoundErrors is falsy', () => { + expect(isSuppressedFailure({status: 'fail', args: [{suppressNotFoundErrors: false}]})).to.be.false; + expect(isSuppressedFailure({status: 'fail', args: [{suppressNotFoundErrors: 'true'}]})).to.be.false; + }); + +});