From 622cea091a0d67527f840301887f024655a28f32 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Thu, 25 Jun 2026 18:20:19 +0200 Subject: [PATCH 1/3] Artificially fail tests to test reporting. --- .github/workflows/build.yaml | 8 +++--- .../src/test/vscode-suite/dsl.test.ts | 26 +++---------------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 19d69eec1..b616c7fa5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -50,7 +50,7 @@ jobs: strategy: matrix: os: [ubicloud-standard-4, windows-latest, macos-latest] - fail-fast: true + fail-fast: false env: CODE_VERSION: "1.100.0" runs-on: ${{ matrix.os }} @@ -96,7 +96,7 @@ jobs: DELAY_FACTOR: 10 RASCAL_LSP_DEV_DEPLOY: true _JAVA_OPTIONS: '-Xmx5G' - run: npx extest setup-and-run out/test/vscode-suite/*.test.js --storage uitests + run: npx extest setup-and-run out/test/vscode-suite/dsl.test.js --storage uitests - name: "UI test (mac)" shell: bash @@ -106,7 +106,7 @@ jobs: DELAY_FACTOR: 15 RASCAL_LSP_DEV_DEPLOY: true _JAVA_OPTIONS: '-Xmx5G' - run: npx extest setup-and-run out/test/vscode-suite/*.test.js --storage uitests + run: npx extest setup-and-run out/test/vscode-suite/dsl.test.js --storage uitests - name: "UI test (ubuntu)" shell: bash @@ -118,7 +118,7 @@ jobs: _JAVA_OPTIONS: '-Xmx5G' # we have 16gb of memory, make sure LSP, REPL & DSL-LSP can start run: | sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 # workaround for issue with Ubuntu 24.04 https://github.com/redhat-developer/vscode-extension-tester/blob/main/KNOWN_ISSUES.md#openresources-not-working-with-apparmor-2404 - xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' npx extest setup-and-run out/test/vscode-suite/*.test.js --storage uitests + xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' npx extest setup-and-run out/test/vscode-suite/dsl.test.js --storage uitests - name: Upload Screenshots uses: actions/upload-artifact@v7 diff --git a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts index 72eec55e6..8d0c79ed1 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts @@ -32,6 +32,7 @@ import { expect } from 'chai'; import * as fs from 'fs/promises'; import { Suite } from 'mocha'; import * as path from 'path/posix'; +import { fail } from 'assert'; function parameterizedDescribe(body: (this: Suite, errorRecovery: boolean) => void) { describe('DSL', function() { body.apply(this, [false]); }); @@ -103,28 +104,9 @@ parameterizedDescribe(function (errorRecovery: boolean) { await protectedFiles.restore(); }); - it("has highlighting and parse errors", async function () { - await ignoreFails(new Workbench().getEditorView().closeAllEditors()); - const editor = await ide.openModule(TestWorkspace.picoFile); - const isPicoLoading = ide.statusContains("Pico"); - // we might miss this event, but we wait for it to show up - await ignoreFails(driver.wait(isPicoLoading, Delays.normal, "Pico parser generator should have started")); - // now wait for the Pico parser generator to disappear - await driver.wait(async () => !(await isPicoLoading()), Delays.verySlow, "Pico parser generator should have finished", 100); - await ide.hasSyntaxHighlighting(editor, Delays.slow); - console.log("We got syntax highlighting"); - try { - await editor.setTextAtLine(10, "b := ;"); - await ide.hasErrorSquiggly(editor, Delays.slow); - } catch (e) { - console.log(`Failed to trigger parse error: ${e}`); - if (e instanceof Error) { - console.log(e.stack); - } - } finally { - await ide.revertOpenChanges(); - } - }).retries(2); + it.only("has highlighting and parse errors", async function () { + fail("Artificial failure"); + }); it("has highlighting and parse errors for second extension", async function () { const editor = await ide.openModule(TestWorkspace.picoNewFile); From 49528ab432e830788460a4026bec3092cd9eeeaa Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Thu, 25 Jun 2026 18:19:35 +0200 Subject: [PATCH 2/3] Take failed test logs from file. --- .../src/test/vscode-suite/utils.ts | 89 +++++++++++++------ 1 file changed, 61 insertions(+), 28 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index 6bdffe9c6..b5a42cdb7 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -27,7 +27,8 @@ import { assert, expect } from "chai"; import { createHash } from "crypto"; -import { readFile, stat, unlink, writeFile } from "fs/promises"; +import { existsSync } from "fs"; +import { readdir, readFile, stat, unlink, writeFile } from "fs/promises"; import * as os from 'os'; import path from "path/posix"; import { env } from "process"; @@ -524,40 +525,72 @@ async function assureDebugLevelLoggingIsEnabled() { export function printRascalOutputOnFailure(channel: 'Language Parametric Rascal' | 'Rascal MPL') { const ZOOM_OUT_FACTOR = 5; + const N_LOG_LINES = 250; + // We guess some locations where the logs can be, since we cannot easily retrieve those from VS Code. + const TEST_STORAGE_DIRS = [ + path.join(path.resolve(), "uitests"), // typical CI path + path.join(os.tmpdir(), "vscode-uitests") // typical local path + ]; + afterEach("print output in case of failure", async function () { if (!this.currentTest || this.currentTest.state !== "failed") { return; } - const bbp = new BottomBarPanel(); - try { - for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { - await new Workbench().executeCommand('workbench.action.zoomOut'); - } - await bbp.maximize(); - console.log('**********************************************'); - console.log('***** Rascal MPL output for the failed tests: '); - let textLines: WebElement[] = []; - let tries = 0; - while (textLines.length === 0 && tries < 3) { - await showRascalOutput(bbp, channel); - textLines = await ignoreFails(bbp.findElements(By.className('view-line'))) ?? []; - tries++; - } - if (textLines.length === 0) { - console.log("We could not capture the output lines"); - } - for (const l of textLines) { - console.log(await l.getText()); + console.log('**********************************************'); + console.log(`***** ${channel} output for the failed tests: `); + + let foundLogs = false; + for (const vsCodeDir of TEST_STORAGE_DIRS) { + try { + const logDir = path.join(vsCodeDir, "settings", "logs"); + if (existsSync(logDir)) { + for (const entry of await readdir(logDir, {recursive: true})) { + if (entry.includes("usethesource.rascalmpl") && entry.includes(channel)) { + console.log(`***** ${entry}`); + const contents = await readFile(path.join(logDir, entry), {encoding: "utf-8"}); + console.log(contents.split('\n').splice(-N_LOG_LINES).join('\n')); + foundLogs = true; + } + } + } + } catch (e) { + console.log(`Error capturing logs in ${vsCodeDir}: `, e); } - } catch (e) { - console.log('Error capturing output: ', e); } - finally { - console.log('*******End output*****************************'); - for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { - await new Workbench().executeCommand('workbench.action.zoomIn'); + + if (!foundLogs) { + // Fall back to legacy copy-from-VS Code approach if there were no log files. + const bbp = new BottomBarPanel(); + try { + for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { + await new Workbench().executeCommand('workbench.action.zoomOut'); + } + await bbp.maximize(); + let textLines: WebElement[] = []; + let tries = 0; + while (textLines.length === 0 && tries < 3) { + await showRascalOutput(bbp, channel); + textLines = await ignoreFails(bbp.findElements(By.className('view-line'))) ?? []; + tries++; + } + if (textLines.length === 0) { + console.log("We could not capture the output lines"); + } + + for (const l of textLines) { + console.log(await l.getText()); + } + } catch (e) { + console.log('Error capturing output: ', e); + } + finally { + for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { + await new Workbench().executeCommand('workbench.action.zoomIn'); + } + await bbp.closePanel(); } - await bbp.closePanel(); } + + console.log('*******End output*****************************'); }); } From 570d288204bc501e2e0745a21d55402ae61fb4b0 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Fri, 26 Jun 2026 14:29:04 +0200 Subject: [PATCH 3/3] Revert "Artificially fail tests to test reporting." This reverts commit 622cea091a0d67527f840301887f024655a28f32. --- .github/workflows/build.yaml | 8 +++--- .../src/test/vscode-suite/dsl.test.ts | 26 ++++++++++++++++--- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b616c7fa5..19d69eec1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -50,7 +50,7 @@ jobs: strategy: matrix: os: [ubicloud-standard-4, windows-latest, macos-latest] - fail-fast: false + fail-fast: true env: CODE_VERSION: "1.100.0" runs-on: ${{ matrix.os }} @@ -96,7 +96,7 @@ jobs: DELAY_FACTOR: 10 RASCAL_LSP_DEV_DEPLOY: true _JAVA_OPTIONS: '-Xmx5G' - run: npx extest setup-and-run out/test/vscode-suite/dsl.test.js --storage uitests + run: npx extest setup-and-run out/test/vscode-suite/*.test.js --storage uitests - name: "UI test (mac)" shell: bash @@ -106,7 +106,7 @@ jobs: DELAY_FACTOR: 15 RASCAL_LSP_DEV_DEPLOY: true _JAVA_OPTIONS: '-Xmx5G' - run: npx extest setup-and-run out/test/vscode-suite/dsl.test.js --storage uitests + run: npx extest setup-and-run out/test/vscode-suite/*.test.js --storage uitests - name: "UI test (ubuntu)" shell: bash @@ -118,7 +118,7 @@ jobs: _JAVA_OPTIONS: '-Xmx5G' # we have 16gb of memory, make sure LSP, REPL & DSL-LSP can start run: | sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 # workaround for issue with Ubuntu 24.04 https://github.com/redhat-developer/vscode-extension-tester/blob/main/KNOWN_ISSUES.md#openresources-not-working-with-apparmor-2404 - xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' npx extest setup-and-run out/test/vscode-suite/dsl.test.js --storage uitests + xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' npx extest setup-and-run out/test/vscode-suite/*.test.js --storage uitests - name: Upload Screenshots uses: actions/upload-artifact@v7 diff --git a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts index 8d0c79ed1..72eec55e6 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts @@ -32,7 +32,6 @@ import { expect } from 'chai'; import * as fs from 'fs/promises'; import { Suite } from 'mocha'; import * as path from 'path/posix'; -import { fail } from 'assert'; function parameterizedDescribe(body: (this: Suite, errorRecovery: boolean) => void) { describe('DSL', function() { body.apply(this, [false]); }); @@ -104,9 +103,28 @@ parameterizedDescribe(function (errorRecovery: boolean) { await protectedFiles.restore(); }); - it.only("has highlighting and parse errors", async function () { - fail("Artificial failure"); - }); + it("has highlighting and parse errors", async function () { + await ignoreFails(new Workbench().getEditorView().closeAllEditors()); + const editor = await ide.openModule(TestWorkspace.picoFile); + const isPicoLoading = ide.statusContains("Pico"); + // we might miss this event, but we wait for it to show up + await ignoreFails(driver.wait(isPicoLoading, Delays.normal, "Pico parser generator should have started")); + // now wait for the Pico parser generator to disappear + await driver.wait(async () => !(await isPicoLoading()), Delays.verySlow, "Pico parser generator should have finished", 100); + await ide.hasSyntaxHighlighting(editor, Delays.slow); + console.log("We got syntax highlighting"); + try { + await editor.setTextAtLine(10, "b := ;"); + await ide.hasErrorSquiggly(editor, Delays.slow); + } catch (e) { + console.log(`Failed to trigger parse error: ${e}`); + if (e instanceof Error) { + console.log(e.stack); + } + } finally { + await ide.revertOpenChanges(); + } + }).retries(2); it("has highlighting and parse errors for second extension", async function () { const editor = await ide.openModule(TestWorkspace.picoNewFile);