diff --git a/JetStreamDriver.js b/JetStreamDriver.js index abed590e..4440b455 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -612,9 +612,7 @@ class Benchmark { constructor(plan) { this.plan = plan; - this.tags = new Set(plan.tags.map(each => each.toLowerCase())); - if (this.tags.size != plan.tags.length) - throw new Error(`${this.name} got duplicate tags: ${plan.tags.join()}`); + this.tags = this.processTags(plan.tags) this.iterations = getIterationCount(plan); this.isAsync = !!plan.isAsync; this.scripts = null; @@ -622,6 +620,16 @@ class Benchmark { this._state = BenchmarkState.READY; } + processTags(rawTags) { + const tags = new Set(rawTags.map(each => each.toLowerCase())); + if (tags.size != rawTags.length) + throw new Error(`${this.name} got duplicate tags: ${rawTags.join()}`); + tags.add("all"); + if (!tags.has("default")) + tags.add("disabled"); + return tags; + } + get name() { return this.plan.name; } get isDone() { @@ -2184,22 +2192,23 @@ let BENCHMARKS = [ }) ]; -// LuaJSFight tests -const luaJSFightTests = [ - "hello_world" - , "list_search" - , "lists" - , "string_lists" -]; -for (const test of luaJSFightTests) { - BENCHMARKS.push(new DefaultBenchmark({ - name: `${test}-LJF`, - files: [ - `./LuaJSFight/${test}.js` - ], - tags: ["LuaJSFight"], - })); -} +// FIXME: figure out what to do this these benchmarks. +// // LuaJSFight tests +// const luaJSFightTests = [ +// "hello_world" +// , "list_search" +// , "lists" +// , "string_lists" +// ]; +// for (const test of luaJSFightTests) { +// BENCHMARKS.push(new DefaultBenchmark({ +// name: `${test}-LJF`, +// files: [ +// `./LuaJSFight/${test}.js` +// ], +// tags: ["LuaJSFight"], +// })); +// } // SunSpider tests const SUNSPIDER_TESTS = [ @@ -2263,8 +2272,6 @@ for (const benchmark of BENCHMARKS) { else benchmarksByName.set(name, benchmark); - benchmark.tags.add("all"); - for (const tag of benchmark.tags) { if (benchmarksByTag.has(tag)) benchmarksByTag.get(tag).push(benchmark); diff --git a/tests/helper.mjs b/tests/helper.mjs new file mode 100644 index 00000000..0867e8a2 --- /dev/null +++ b/tests/helper.mjs @@ -0,0 +1,82 @@ +import { styleText } from "node:util"; +import core from "@actions/core"; +import commandLineUsage from "command-line-usage"; + +export const GITHUB_ACTIONS_OUTPUT = "GITHUB_ACTIONS_OUTPUT" in process.env; + +export function logInfo(...args) { + const text = args.join(" ") + if (GITHUB_ACTIONS_OUTPUT) + core.info(styleText("yellow", text)); + else + console.log(styleText("yellow", text)); +} + +export function logError(...args) { + let error; + if (args.length == 1 && args[0] instanceof Error) + error = args[0]; + const text = args.join(" "); + if (GITHUB_ACTIONS_OUTPUT) { + if (error?.stack) + core.error(error.stack); + else + core.error(styleText("red", text)); + } else { + if (error?.stack) + console.error(styleText("red", error.stack)); + else + console.error(styleText("red", text)); + } +} + +export async function logGroup(name, body) { + if (GITHUB_ACTIONS_OUTPUT) { + core.startGroup(name); + } else { + logInfo("=".repeat(80)); + logInfo(name); + logInfo(".".repeat(80)); + } + try { + return await body(); + } finally { + if (GITHUB_ACTIONS_OUTPUT) + core.endGroup(); + } +} + + +export function printHelp(message = "", optionDefinitions) { + const usage = commandLineUsage([ + { + header: "Run all tests", + }, + { + header: "Options", + optionList: optionDefinitions, + }, + ]); + if (!message) { + console.log(usage); + process.exit(0); + } else { + console.error(message); + console.error(); + console.error(usage); + process.exit(1); + } +} + + +export async function runTest(label, testFunction) { + try { + await logGroup(label, testFunction); + logInfo("āœ… Test completed!"); + } catch(e) { + logError("āŒ Test failed!"); + logError(e); + return false; + } + return true; +} diff --git a/tests/run-shell.mjs b/tests/run-shell.mjs index 213a37ee..f31a646b 100644 --- a/tests/run-shell.mjs +++ b/tests/run-shell.mjs @@ -1,49 +1,29 @@ #! /usr/bin/env node import commandLineArgs from "command-line-args"; -import commandLineUsage from "command-line-usage"; import { spawnSync } from "child_process"; import { fileURLToPath } from "url"; import { styleText } from "node:util"; import * as path from "path"; import * as fs from "fs"; import * as os from "os"; -import core from "@actions/core" +import core from "@actions/core"; + +import {logInfo, logError, logGroup, printHelp, runTest, GITHUB_ACTIONS_OUTPUT} from "./helper.mjs"; const optionDefinitions = [ { name: "shell", type: String, description: "Set the shell to test, choices are [jsc, v8, spidermonkey]." }, { name: "help", alias: "h", description: "Print this help text." }, ]; -function printHelp(message = "") { - const usage = commandLineUsage([ - { - header: "Run all tests", - }, - { - header: "Options", - optionList: optionDefinitions, - }, - ]); - if (!message) { - console.log(usage); - process.exit(0); - } else { - console.error(message); - console.error(); - console.error(usage); - process.exit(1); - } -} - const options = commandLineArgs(optionDefinitions); if ("help" in options) - printHelp(); + printHelp(optionDefinitions); const JS_SHELL= options?.shell; if (!JS_SHELL) - printHelp("No javascript shell specified, use --shell"); + printHelp("No javascript shell specified, use --shell", optionDefinitions); const SHELL_NAME = (function() { switch (JS_SHELL) { @@ -70,49 +50,16 @@ const UNIT_TEST_PATH = path.join(SRC_DIR, "tests", "unit-tests.js"); function convertCliArgs(cli, ...cliArgs) { if (SHELL_NAME == "spidermonkey") - return [cli, ...cliArgs] + return [cli, ...cliArgs]; return [cli, "--", ...cliArgs]; } -const GITHUB_ACTIONS_OUTPUT = "GITHUB_ACTIONS_OUTPUT" in process.env; - -function log(...args) { - const text = args.join(" ") - if (GITHUB_ACTIONS_OUTPUT) - core.info(styleText("yellow", text)) - else - console.log(styleText("yellow", text)) -} - -function logError(...args) { - const text = args.join(" ") - if (GITHUB_ACTIONS_OUTPUT) - core.error(styleText("red", text)) - else - console.error(styleText("red", text)) -} - -function logGroup(name, body) { - if (GITHUB_ACTIONS_OUTPUT) { - core.startGroup(name); - } else { - log("=".repeat(80)) - log(name); - log(".".repeat(80)) - } - try { - return body(); - } finally { - if (GITHUB_ACTIONS_OUTPUT) - core.endGroup(); - } -} const SPAWN_OPTIONS = { stdio: ["inherit", "inherit", "inherit"] }; -function sh(binary, args) { +function sh(binary, ...args) { const cmd = `${binary} ${args.join(" ")}`; if (GITHUB_ACTIONS_OUTPUT) { core.startGroup(binary); @@ -128,21 +75,19 @@ function sh(binary, args) { } } finally { if (GITHUB_ACTIONS_OUTPUT) - core.endGroup() + core.endGroup(); } } async function runTests() { - const shellBinary = logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); + const shellBinary = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); let success = true; - success &&= runTest("Run UnitTests", () => sh(shellBinary, [UNIT_TEST_PATH])); - success &&= runTest("Run Complete Suite", () => sh(shellBinary, convertCliArgs(CLI_PATH))); - success &&= runTest("Run Single Suite", () => { - sh(shellBinary, convertCliArgs(CLI_PATH, "proxy-mobx")); - }); - if (!success) { - process.exit(1) - } + success &&= await runTest("Run UnitTests", () => sh(shellBinary, UNIT_TEST_PATH)); + success &&= await runCLITest("Run Single Suite", shellBinary, "proxy-mobx"); + success &&= await runCLITest("Run Disabled Suite", shellBinary, "disabled"); + success &&= await runCLITest("Run Default Suite", shellBinary); + if (!success) + process.exit(1); } function jsvuOSName() { @@ -161,30 +106,24 @@ function jsvuOSName() { default: throw new Error("Unsupported architecture"); } }; - return `${osName()}${osArch()}` + return `${osName()}${osArch()}`; } const DEFAULT_JSC_LOCATION = "/System/Library/Frameworks/JavaScriptCore.framework/Versions/Current/Helpers/jsc" function testSetup() { - sh("jsvu", [`--engines=${SHELL_NAME}`, `--os=${jsvuOSName()}`]); + sh("jsvu", `--engines=${SHELL_NAME}`, `--os=${jsvuOSName()}`); let shellBinary = path.join(os.homedir(), ".jsvu/bin", SHELL_NAME); if (!fs.existsSync(shellBinary) && SHELL_NAME == "javascriptcore") - shellBinary = DEFAULT_JSC_LOCATION + shellBinary = DEFAULT_JSC_LOCATION; if (!fs.existsSync(shellBinary)) throw new Error(`Could not find shell binary: ${shellBinary}`); - log(`Installed JavaScript Shell: ${shellBinary}`); - return shellBinary + logInfo(`Installed JavaScript Shell: ${shellBinary}`); + return shellBinary; } -function runTest(testName, test) { - try { - logGroup(testName, test) - } catch(e) { - logError("TEST FAILED") - return false - } - return true +function runCLITest(name, shellBinary, ...args) { + return runTest(name, () => sh(shellBinary, ...convertCliArgs(CLI_PATH, ...args))); } setImmediate(runTests); diff --git a/tests/run.mjs b/tests/run.mjs index 205a65f0..6ef609a9 100644 --- a/tests/run.mjs +++ b/tests/run.mjs @@ -3,7 +3,8 @@ import serve from "./server.mjs"; import { Builder, Capabilities } from "selenium-webdriver"; import commandLineArgs from "command-line-args"; -import commandLineUsage from "command-line-usage"; + +import {logInfo, logError, printHelp, runTest} from "./helper.mjs"; const optionDefinitions = [ { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome, edge]. By default the $BROWSER env variable is used." }, @@ -11,35 +12,15 @@ const optionDefinitions = [ { name: "help", alias: "h", description: "Print this help text." }, ]; -function printHelp(message = "") { - const usage = commandLineUsage([ - { - header: "Run all tests", - }, - { - header: "Options", - optionList: optionDefinitions, - }, - ]); - if (!message) { - console.log(usage); - process.exit(0); - } else { - console.error(message); - console.error(); - console.error(usage); - process.exit(1); - } -} const options = commandLineArgs(optionDefinitions); if ("help" in options) - printHelp(); + printHelp(optionDefinitions); const BROWSER = options?.browser; if (!BROWSER) - printHelp("No browser specified, use $BROWSER or --browser"); + printHelp("No browser specified, use $BROWSER or --browser", optionDefinitions); let capabilities; switch (BROWSER) { @@ -65,24 +46,43 @@ switch (BROWSER) { } process.on("unhandledRejection", (err) => { - console.error(err); + logError(err); process.exit(1); }); process.once("uncaughtException", (err) => { - console.error(err); + logError(err); process.exit(1); }); const PORT = options.port; const server = await serve(PORT); -async function testEnd2End() { +async function runTests() { + let success = true; + try { + success &&= await runTest("Run Single Suite", () => testEnd2End({ test: "proxy-mobx" })); + success &&= await runTest("Run Disabled Suite", () => testEnd2End({ tag: "disabled" })); + success &&= await runTest("Run Default Suite", () => testEnd2End()); + } finally { + server.close(); + } + if (!success) + process.exit(1); +} + + +async function testEnd2End(params) { const driver = await new Builder().withCapabilities(capabilities).build(); + const urlParams = Object.assign({ + worstCaseCount: 2, + iterationCount: 3 + }, params); let results; try { - const url = `http://localhost:${PORT}/index.html?worstCaseCount=2&iterationCount=3`; - console.log(`JetStream PREPARE ${url}`); - await driver.get(url); + const url = new URL(`http://localhost:${PORT}/index.html`); + url.search = new URLSearchParams(urlParams).toString(); + logInfo(`JetStream PREPARE ${url}`); + await driver.get(url.toString()); await driver.executeAsyncScript((callback) => { // callback() is explicitly called without the default event // as argument to avoid serialization issues with chromedriver. @@ -90,23 +90,19 @@ async function testEnd2End() { // We might not get a chance to install the on-ready listener, thus // we also check if the runner is ready synchronously. if (globalThis?.JetStream?.isReady) - callback() + callback(); }); results = await benchmarkResults(driver); // FIXME: validate results; - console.log("\nāœ… Tests completed!"); } catch(e) { - console.error("\nāŒ Tests failed!"); - console.error(e); throw e; } finally { driver.quit(); - server.close(); } } async function benchmarkResults(driver) { - console.log("JetStream START"); + logInfo("JetStream START"); await driver.manage().setTimeouts({ script: 60_000 }); await driver.executeAsyncScript((callback) => { globalThis.JetStream.start(); @@ -160,4 +156,4 @@ function logIncrementalResult(previousResults, benchmarkResults) { } } -setImmediate(testEnd2End); +setImmediate(runTests); diff --git a/tests/unit-tests.js b/tests/unit-tests.js index f8efd23a..a1533f69 100644 --- a/tests/unit-tests.js +++ b/tests/unit-tests.js @@ -19,9 +19,12 @@ function assertEquals(actual, expected, message) { } } -(function testTagsAreStrings() { +(function testTagsAreLowerCaseStrings() { for (const benchmark of BENCHMARKS) { - benchmark.tags.forEach(tag => assertTrue(typeof(tag) == "string")) + benchmark.tags.forEach(tag => { + assertTrue(typeof(tag) == "string"); + assertTrue(tag == tag.toLowerCase()); + }) } })(); @@ -29,10 +32,12 @@ function assertEquals(actual, expected, message) { (function testTagsAll() { for (const benchmark of BENCHMARKS) { - assertTrue(benchmark.tags instanceof Set); - assertTrue(benchmark.tags.size > 0); - assertTrue(benchmark.tags.has("all")); - assertFalse(benchmark.tags.has("All")); + const tags = benchmark.tags; + assertTrue(tags instanceof Set); + assertTrue(tags.size > 0); + assertTrue(tags.has("all")); + assertFalse(tags.has("All")); + assertTrue(tags.has("default") ^ tags.has("disabled")); } })();