From 2f0c36aebde34ff2c23e1e18d3487d1dd4be6665 Mon Sep 17 00:00:00 2001 From: Aakash Hotchandani Date: Wed, 6 May 2026 21:14:44 +0530 Subject: [PATCH] fix(automate): mark every Automate session in describe/it suites that call browser.end() between tests When a Nightwatch test file uses BDD `describe`/`it` with `browser.end()` in `afterEach`, only the *last* session of the file was being marked on the Automate dashboard. Earlier sessions stayed at status=`done` (untested). Two distinct issues were fixed: 1. Session marking on the Automate dashboard - Nightwatch's built-in transport only PUTs the session status via `sendReasonToBrowserstack` once, from `testSuiteFinished`. When tests call `browser.end()` between tests, every session except the last one ends without ever being marked. - Fix: in the plugin's `TestRunFinished` event handler, call `browserstack_executor: setSessionStatus` against the live session before the user's `afterEach` (and any `browser.end()`) runs. Nightwatch core's later GET-then-PUT respects the existing status, so the last session is not double-marked. 2. Stale `session_id` in observability TestRun events - `reporter.testResults.sessionId` is populated only once per file via `setSessionInfo()` in `startTestSuite()`. When subsequent tests get a new BrowserStack session (after `browser.end()`), the metadata still points at the first session, so observability events for test 2+ were tagged with the wrong `integrations.browserstack.session_id`. - The async `TestRunFinished` event handler can also race the user's `afterEach`: in some envs `browser.sessionId` is already null by the time the handler runs. - Fix: snapshot `(sessionId, capabilities)` at `TestRunStarted` time (when the session is guaranteed live), store on `TestMap.activeTestRuns`, and resolve in `sendTestRunEvent` with priority: 1. snapshot (always set, reliable) 2. live `browser.sessionId` 3. metadata fallback Verified against `npm run sample-test` (2 envs * 2 tests each in one `describe`): Before: env1/env2 Login -> done, Locked -> passed After : all 4 sessions -> passed, each with its own correct session_id Also: surface the actual response body when build start fails (was printing `[object Object]` because `makeRequest` rejects with a plain object on non-2xx). Co-Authored-By: Claude Opus 4.7 (1M context) --- nightwatch/globals.js | 35 ++++++++++++++++++++++++++++++++++- src/testObservability.js | 22 +++++++++++++++++++++- src/utils/testMap.js | 31 +++++++++++++++++++++++++++---- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index c0549d2..f6852da 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -277,6 +277,14 @@ module.exports = { if (testRunner !== 'cucumber'){ const uuid = TestMap.storeTestDetails(test); process.env.TEST_RUN_UUID = uuid; + // Capture the live session id at TestRunStarted (when browser.end() + // hasn't run yet). TestRunFinished's async handler can race with + // afterEach: by the time it executes, browser.sessionId may already + // be null. The snapshot guarantees the finish event is tagged with + // the session that actually ran the test. + if (typeof browser !== 'undefined' && browser.sessionId) { + TestMap.setSessionSnapshot(uuid, browser.sessionId, browser.capabilities); + } testEventPromises.push(testObservability.sendTestRunEvent('TestRunStarted', test, uuid)); } }); @@ -289,6 +297,31 @@ module.exports = { return; } try { + // Mark the live BrowserStack session with pass/fail BEFORE the user's + // afterEach runs. Nightwatch's built-in transport only marks the LAST + // session of a suite via `sendReasonToBrowserstack`, so when a test + // calls `browser.end()` in afterEach, all earlier sessions stay + // unmarked ("done") on the Automate dashboard. We use the same + // browserstack_executor action the user would call manually. + if (typeof browser !== 'undefined' && browser.sessionId) { + try { + const testName = test?.testcase; + const eventData = (testName && test?.envelope?.[testName]?.testcase) || null; + const failedCommand = eventData?.commands && Array.isArray(eventData.commands) + ? eventData.commands.find(cmd => cmd.status === 'fail') + : null; + const status = failedCommand ? 'failed' : 'passed'; + let reason = ''; + if (failedCommand && failedCommand.result) { + reason = (failedCommand.result.message || failedCommand.result.stack || 'Test failed').toString().slice(0, 280); + } + const payload = JSON.stringify({status, reason}); + await browser.execute(`browserstack_executor: {"action": "setSessionStatus", "arguments": ${payload}}`); + } catch (statusErr) { + Logger.debug(`Could not set BrowserStack session status: ${statusErr.message || statusErr}`); + } + } + await accessibilityAutomation.afterEachExecution(test, uuid); if (testRunner !== 'cucumber'){ // Drain pending tags synchronously before the async sendTestRunEvent, @@ -301,7 +334,7 @@ module.exports = { // (test body runs before TestRunStarted assigns the new UUID) delete process.env.TEST_RUN_UUID; } - + } catch (error) { Logger.error(`Error in TestRunFinished event: ${error.message}`); TestMap.markTestFinished(uuid); diff --git a/src/testObservability.js b/src/testObservability.js index fa288cd..dcd336e 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -169,6 +169,9 @@ class TestObservability { } catch (error) { if (error.response) { Logger.error(`EXCEPTION IN BUILD START EVENT : ${error.response.status} ${error.response.statusText} ${JSON.stringify(error.response.data)}`); + } else if (error && typeof error === 'object' && !error.message) { + // makeRequest rejects with response.body (a plain object) on non-2xx — surface its contents. + Logger.error(`EXCEPTION IN BUILD START EVENT : ${JSON.stringify(error)}`); } else { Logger.error(`EXCEPTION IN BUILD START EVENT : ${error.message || error}`); } @@ -483,6 +486,23 @@ class TestObservability { const testResults = {}; const testBody = this.getTestBody(test.testCaseData); const provider = helper.getCloudProvider(testMetaData.host); + // testMetaData.sessionId / sessionCapabilities are populated only once per test + // suite (via reporter.setSessionInfo() in startTestSuite()). When tests use + // browser.end() in afterEach, every subsequent test runs against a NEW + // BrowserStack session whose id is NOT reflected back into the reporter's + // metadata. Resolve the right session id with this priority: + // 1. Snapshot taken at TestRunStarted (always set, captured before any + // afterEach runs — this is the session the test actually ran in). + // 2. Live `browser.sessionId` (correct if the async handler beats + // afterEach's browser.end()). + // 3. metadata.sessionId fallback (only correct for the very first test + // in the suite). + const snapshot = TestMap.getSessionSnapshot(uuid); + const liveSessionId = snapshot?.sessionId + || ((typeof browser !== 'undefined' && browser.sessionId) ? browser.sessionId : testMetaData.sessionId); + const liveSessionCapabilities = (snapshot?.sessionCapabilities && Object.keys(snapshot.sessionCapabilities).length > 0) + ? snapshot.sessionCapabilities + : ((typeof browser !== 'undefined' && browser.capabilities && Object.keys(browser.capabilities || {}).length > 0) ? browser.capabilities : testMetaData.sessionCapabilities); const testData = { uuid: uuid, type: 'test', @@ -504,7 +524,7 @@ class TestObservability { result: 'pending', framework: 'nightwatch', integrations: { - [provider]: helper.getIntegrationsObject(testMetaData.sessionCapabilities, testMetaData.sessionId, testMetaData.host, settings?.desiredCapabilities?.['bstack:options']?.osVersion) + [provider]: helper.getIntegrationsObject(liveSessionCapabilities, liveSessionId, testMetaData.host, settings?.desiredCapabilities?.['bstack:options']?.osVersion) }, product_map: { observability: helper.isTestObservabilitySession(), diff --git a/src/utils/testMap.js b/src/utils/testMap.js index 39b774c..1c95c0e 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -9,7 +9,7 @@ class TestMap { static storeTestDetails(test) { const testIdentifier = this.generateTestIdentifier(test); const uuid = this.generateUUID(); - + if (!sharedTestMap.has(testIdentifier)) { sharedTestMap.set(testIdentifier, { baseUuid: uuid, // Store the first UUID as base @@ -28,19 +28,42 @@ class TestMap { testData.currentUuid = uuid; // Update to latest UUID sharedTestMap.set(testIdentifier, testData); } - + // Track this as an active test run activeTestRuns.set(uuid, { identifier: testIdentifier, startedAt: new Date().toISOString(), hasFinished: false }); - + sharedCurrentTest = testIdentifier; - + return uuid; } + // Snapshot the live BrowserStack session id and capabilities for a uuid at + // TestRunStarted time. The reporter's metadata.sessionId only reflects the + // first session of the suite, and `browser.sessionId` may already be null + // by the time TestRunFinished fires (afterEach calling browser.end() can + // race the async event handler). Storing the snapshot lets TestRunFinished + // reach back to the correct session id even after the session has been + // ended. + static setSessionSnapshot(uuid, sessionId, capabilities) { + if (!uuid || !activeTestRuns.has(uuid)) {return} + const run = activeTestRuns.get(uuid); + run.sessionId = sessionId; + run.sessionCapabilities = capabilities; + activeTestRuns.set(uuid, run); + } + + static getSessionSnapshot(uuid) { + if (!uuid || !activeTestRuns.has(uuid)) {return null} + const run = activeTestRuns.get(uuid); + if (!run.sessionId) {return null} + + return {sessionId: run.sessionId, sessionCapabilities: run.sessionCapabilities}; + } + static getUUID(test = null) { if (test) { const testIdentifier = typeof test === 'string' ? test : this.generateTestIdentifier(test);