diff --git a/nightwatch/globals.js b/nightwatch/globals.js index c0549d2..f6852da 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -277,6 +277,14 @@ module.exports = { if (testRunner !== 'cucumber'){ const uuid = TestMap.storeTestDetails(test); process.env.TEST_RUN_UUID = uuid; + // Capture the live session id at TestRunStarted (when browser.end() + // hasn't run yet). TestRunFinished's async handler can race with + // afterEach: by the time it executes, browser.sessionId may already + // be null. The snapshot guarantees the finish event is tagged with + // the session that actually ran the test. + if (typeof browser !== 'undefined' && browser.sessionId) { + TestMap.setSessionSnapshot(uuid, browser.sessionId, browser.capabilities); + } testEventPromises.push(testObservability.sendTestRunEvent('TestRunStarted', test, uuid)); } }); @@ -289,6 +297,31 @@ module.exports = { return; } try { + // Mark the live BrowserStack session with pass/fail BEFORE the user's + // afterEach runs. Nightwatch's built-in transport only marks the LAST + // session of a suite via `sendReasonToBrowserstack`, so when a test + // calls `browser.end()` in afterEach, all earlier sessions stay + // unmarked ("done") on the Automate dashboard. We use the same + // browserstack_executor action the user would call manually. + if (typeof browser !== 'undefined' && browser.sessionId) { + try { + const testName = test?.testcase; + const eventData = (testName && test?.envelope?.[testName]?.testcase) || null; + const failedCommand = eventData?.commands && Array.isArray(eventData.commands) + ? eventData.commands.find(cmd => cmd.status === 'fail') + : null; + const status = failedCommand ? 'failed' : 'passed'; + let reason = ''; + if (failedCommand && failedCommand.result) { + reason = (failedCommand.result.message || failedCommand.result.stack || 'Test failed').toString().slice(0, 280); + } + const payload = JSON.stringify({status, reason}); + await browser.execute(`browserstack_executor: {"action": "setSessionStatus", "arguments": ${payload}}`); + } catch (statusErr) { + Logger.debug(`Could not set BrowserStack session status: ${statusErr.message || statusErr}`); + } + } + await accessibilityAutomation.afterEachExecution(test, uuid); if (testRunner !== 'cucumber'){ // Drain pending tags synchronously before the async sendTestRunEvent, @@ -301,7 +334,7 @@ module.exports = { // (test body runs before TestRunStarted assigns the new UUID) delete process.env.TEST_RUN_UUID; } - + } catch (error) { Logger.error(`Error in TestRunFinished event: ${error.message}`); TestMap.markTestFinished(uuid); diff --git a/src/testObservability.js b/src/testObservability.js index fa288cd..dcd336e 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -169,6 +169,9 @@ class TestObservability { } catch (error) { if (error.response) { Logger.error(`EXCEPTION IN BUILD START EVENT : ${error.response.status} ${error.response.statusText} ${JSON.stringify(error.response.data)}`); + } else if (error && typeof error === 'object' && !error.message) { + // makeRequest rejects with response.body (a plain object) on non-2xx — surface its contents. + Logger.error(`EXCEPTION IN BUILD START EVENT : ${JSON.stringify(error)}`); } else { Logger.error(`EXCEPTION IN BUILD START EVENT : ${error.message || error}`); } @@ -483,6 +486,23 @@ class TestObservability { const testResults = {}; const testBody = this.getTestBody(test.testCaseData); const provider = helper.getCloudProvider(testMetaData.host); + // testMetaData.sessionId / sessionCapabilities are populated only once per test + // suite (via reporter.setSessionInfo() in startTestSuite()). When tests use + // browser.end() in afterEach, every subsequent test runs against a NEW + // BrowserStack session whose id is NOT reflected back into the reporter's + // metadata. Resolve the right session id with this priority: + // 1. Snapshot taken at TestRunStarted (always set, captured before any + // afterEach runs — this is the session the test actually ran in). + // 2. Live `browser.sessionId` (correct if the async handler beats + // afterEach's browser.end()). + // 3. metadata.sessionId fallback (only correct for the very first test + // in the suite). + const snapshot = TestMap.getSessionSnapshot(uuid); + const liveSessionId = snapshot?.sessionId + || ((typeof browser !== 'undefined' && browser.sessionId) ? browser.sessionId : testMetaData.sessionId); + const liveSessionCapabilities = (snapshot?.sessionCapabilities && Object.keys(snapshot.sessionCapabilities).length > 0) + ? snapshot.sessionCapabilities + : ((typeof browser !== 'undefined' && browser.capabilities && Object.keys(browser.capabilities || {}).length > 0) ? browser.capabilities : testMetaData.sessionCapabilities); const testData = { uuid: uuid, type: 'test', @@ -504,7 +524,7 @@ class TestObservability { result: 'pending', framework: 'nightwatch', integrations: { - [provider]: helper.getIntegrationsObject(testMetaData.sessionCapabilities, testMetaData.sessionId, testMetaData.host, settings?.desiredCapabilities?.['bstack:options']?.osVersion) + [provider]: helper.getIntegrationsObject(liveSessionCapabilities, liveSessionId, testMetaData.host, settings?.desiredCapabilities?.['bstack:options']?.osVersion) }, product_map: { observability: helper.isTestObservabilitySession(), diff --git a/src/utils/testMap.js b/src/utils/testMap.js index 39b774c..1c95c0e 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -9,7 +9,7 @@ class TestMap { static storeTestDetails(test) { const testIdentifier = this.generateTestIdentifier(test); const uuid = this.generateUUID(); - + if (!sharedTestMap.has(testIdentifier)) { sharedTestMap.set(testIdentifier, { baseUuid: uuid, // Store the first UUID as base @@ -28,19 +28,42 @@ class TestMap { testData.currentUuid = uuid; // Update to latest UUID sharedTestMap.set(testIdentifier, testData); } - + // Track this as an active test run activeTestRuns.set(uuid, { identifier: testIdentifier, startedAt: new Date().toISOString(), hasFinished: false }); - + sharedCurrentTest = testIdentifier; - + return uuid; } + // Snapshot the live BrowserStack session id and capabilities for a uuid at + // TestRunStarted time. The reporter's metadata.sessionId only reflects the + // first session of the suite, and `browser.sessionId` may already be null + // by the time TestRunFinished fires (afterEach calling browser.end() can + // race the async event handler). Storing the snapshot lets TestRunFinished + // reach back to the correct session id even after the session has been + // ended. + static setSessionSnapshot(uuid, sessionId, capabilities) { + if (!uuid || !activeTestRuns.has(uuid)) {return} + const run = activeTestRuns.get(uuid); + run.sessionId = sessionId; + run.sessionCapabilities = capabilities; + activeTestRuns.set(uuid, run); + } + + static getSessionSnapshot(uuid) { + if (!uuid || !activeTestRuns.has(uuid)) {return null} + const run = activeTestRuns.get(uuid); + if (!run.sessionId) {return null} + + return {sessionId: run.sessionId, sessionCapabilities: run.sessionCapabilities}; + } + static getUUID(test = null) { if (test) { const testIdentifier = typeof test === 'string' ? test : this.generateTestIdentifier(test);