Skip to content

Commit 76a8438

Browse files
camillobrunicamillobruni
authored
Run all disabled workloads in tests (#103)
- Add "disabled" tag - Move default tags processing to the Benchmark class - Add more tags test - Refactor unittest cli code a bit - Change unittest order to get faster failing feedback (single test, disabled test, default suite) - Disable broken LuaJSFight for now (we should likely just remove them)
1 parent 36e8a98 commit 76a8438

5 files changed

Lines changed: 175 additions & 146 deletions

File tree

‎JetStreamDriver.js‎

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -578,16 +578,24 @@ class Benchmark {
578578
constructor(plan)
579579
{
580580
this.plan = plan;
581-
this.tags = new Set(plan.tags.map(each => each.toLowerCase()));
582-
if (this.tags.size != plan.tags.length)
583-
throw new Error(`${this.name} got duplicate tags: ${plan.tags.join()}`);
581+
this.tags = this.processTags(plan.tags)
584582
this.iterations = getIterationCount(plan);
585583
this.isAsync = !!plan.isAsync;
586584
this.scripts = null;
587585
this.preloads = null;
588586
this._state = BenchmarkState.READY;
589587
}
590588

589+
processTags(rawTags) {
590+
const tags = new Set(rawTags.map(each => each.toLowerCase()));
591+
if (tags.size != rawTags.length)
592+
throw new Error(`${this.name} got duplicate tags: ${rawTags.join()}`);
593+
tags.add("all");
594+
if (!tags.has("default"))
595+
tags.add("disabled");
596+
return tags;
597+
}
598+
591599
get name() { return this.plan.name; }
592600

593601
get isDone() {
@@ -2150,22 +2158,23 @@ let BENCHMARKS = [
21502158
})
21512159
];
21522160

2153-
// LuaJSFight tests
2154-
const luaJSFightTests = [
2155-
"hello_world"
2156-
, "list_search"
2157-
, "lists"
2158-
, "string_lists"
2159-
];
2160-
for (const test of luaJSFightTests) {
2161-
BENCHMARKS.push(new DefaultBenchmark({
2162-
name: `${test}-LJF`,
2163-
files: [
2164-
`./LuaJSFight/${test}.js`
2165-
],
2166-
tags: ["LuaJSFight"],
2167-
}));
2168-
}
2161+
// FIXME: figure out what to do this these benchmarks.
2162+
// // LuaJSFight tests
2163+
// const luaJSFightTests = [
2164+
// "hello_world"
2165+
// , "list_search"
2166+
// , "lists"
2167+
// , "string_lists"
2168+
// ];
2169+
// for (const test of luaJSFightTests) {
2170+
// BENCHMARKS.push(new DefaultBenchmark({
2171+
// name: `${test}-LJF`,
2172+
// files: [
2173+
// `./LuaJSFight/${test}.js`
2174+
// ],
2175+
// tags: ["LuaJSFight"],
2176+
// }));
2177+
// }
21692178

21702179
// SunSpider tests
21712180
const SUNSPIDER_TESTS = [
@@ -2229,8 +2238,6 @@ for (const benchmark of BENCHMARKS) {
22292238
else
22302239
benchmarksByName.set(name, benchmark);
22312240

2232-
benchmark.tags.add("all");
2233-
22342241
for (const tag of benchmark.tags) {
22352242
if (benchmarksByTag.has(tag))
22362243
benchmarksByTag.get(tag).push(benchmark);

‎tests/helper.mjs‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import { styleText } from "node:util";
2+
import core from "@actions/core";
3+
import commandLineUsage from "command-line-usage";
4+
5+
export const GITHUB_ACTIONS_OUTPUT = "GITHUB_ACTIONS_OUTPUT" in process.env;
6+
7+
export function logInfo(...args) {
8+
const text = args.join(" ")
9+
if (GITHUB_ACTIONS_OUTPUT)
10+
core.info(styleText("yellow", text));
11+
else
12+
console.log(styleText("yellow", text));
13+
}
14+
15+
export function logError(...args) {
16+
let error;
17+
if (args.length == 1 && args[0] instanceof Error)
18+
error = args[0];
19+
const text = args.join(" ");
20+
if (GITHUB_ACTIONS_OUTPUT) {
21+
if (error?.stack)
22+
core.error(error.stack);
23+
else
24+
core.error(styleText("red", text));
25+
} else {
26+
if (error?.stack)
27+
console.error(styleText("red", error.stack));
28+
else
29+
console.error(styleText("red", text));
30+
}
31+
}
32+
33+
export async function logGroup(name, body) {
34+
if (GITHUB_ACTIONS_OUTPUT) {
35+
core.startGroup(name);
36+
} else {
37+
logInfo("=".repeat(80));
38+
logInfo(name);
39+
logInfo(".".repeat(80));
40+
}
41+
try {
42+
return await body();
43+
} finally {
44+
if (GITHUB_ACTIONS_OUTPUT)
45+
core.endGroup();
46+
}
47+
}
48+
49+
50+
export function printHelp(message = "", optionDefinitions) {
51+
const usage = commandLineUsage([
52+
{
53+
header: "Run all tests",
54+
},
55+
{
56+
header: "Options",
57+
optionList: optionDefinitions,
58+
},
59+
]);
60+
if (!message) {
61+
console.log(usage);
62+
process.exit(0);
63+
} else {
64+
console.error(message);
65+
console.error();
66+
console.error(usage);
67+
process.exit(1);
68+
}
69+
}
70+
71+
72+
export async function runTest(label, testFunction) {
73+
try {
74+
await logGroup(label, testFunction);
75+
logInfo("✅ Test completed!");
76+
} catch(e) {
77+
logError("❌ Test failed!");
78+
logError(e);
79+
return false;
80+
}
81+
return true;
82+
}

‎tests/run-shell.mjs‎

Lines changed: 22 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,49 +1,29 @@
11
#! /usr/bin/env node
22

33
import commandLineArgs from "command-line-args";
4-
import commandLineUsage from "command-line-usage";
54
import { spawnSync } from "child_process";
65
import { fileURLToPath } from "url";
76
import { styleText } from "node:util";
87
import * as path from "path";
98
import * as fs from "fs";
109
import * as os from "os";
11-
import core from "@actions/core"
10+
import core from "@actions/core";
11+
12+
import {logInfo, logError, logGroup, printHelp, runTest, GITHUB_ACTIONS_OUTPUT} from "./helper.mjs";
1213

1314
const optionDefinitions = [
1415
{ name: "shell", type: String, description: "Set the shell to test, choices are [jsc, v8, spidermonkey]." },
1516
{ name: "help", alias: "h", description: "Print this help text." },
1617
];
1718

18-
function printHelp(message = "") {
19-
const usage = commandLineUsage([
20-
{
21-
header: "Run all tests",
22-
},
23-
{
24-
header: "Options",
25-
optionList: optionDefinitions,
26-
},
27-
]);
28-
if (!message) {
29-
console.log(usage);
30-
process.exit(0);
31-
} else {
32-
console.error(message);
33-
console.error();
34-
console.error(usage);
35-
process.exit(1);
36-
}
37-
}
38-
3919
const options = commandLineArgs(optionDefinitions);
4020

4121
if ("help" in options)
42-
printHelp();
22+
printHelp(optionDefinitions);
4323

4424
const JS_SHELL= options?.shell;
4525
if (!JS_SHELL)
46-
printHelp("No javascript shell specified, use --shell");
26+
printHelp("No javascript shell specified, use --shell", optionDefinitions);
4727

4828
const SHELL_NAME = (function() {
4929
switch (JS_SHELL) {
@@ -70,49 +50,16 @@ const UNIT_TEST_PATH = path.join(SRC_DIR, "tests", "unit-tests.js");
7050

7151
function convertCliArgs(cli, ...cliArgs) {
7252
if (SHELL_NAME == "spidermonkey")
73-
return [cli, ...cliArgs]
53+
return [cli, ...cliArgs];
7454
return [cli, "--", ...cliArgs];
7555
}
7656

77-
const GITHUB_ACTIONS_OUTPUT = "GITHUB_ACTIONS_OUTPUT" in process.env;
78-
79-
function log(...args) {
80-
const text = args.join(" ")
81-
if (GITHUB_ACTIONS_OUTPUT)
82-
core.info(styleText("yellow", text))
83-
else
84-
console.log(styleText("yellow", text))
85-
}
86-
87-
function logError(...args) {
88-
const text = args.join(" ")
89-
if (GITHUB_ACTIONS_OUTPUT)
90-
core.error(styleText("red", text))
91-
else
92-
console.error(styleText("red", text))
93-
}
94-
95-
function logGroup(name, body) {
96-
if (GITHUB_ACTIONS_OUTPUT) {
97-
core.startGroup(name);
98-
} else {
99-
log("=".repeat(80))
100-
log(name);
101-
log(".".repeat(80))
102-
}
103-
try {
104-
return body();
105-
} finally {
106-
if (GITHUB_ACTIONS_OUTPUT)
107-
core.endGroup();
108-
}
109-
}
11057

11158
const SPAWN_OPTIONS = {
11259
stdio: ["inherit", "inherit", "inherit"]
11360
};
11461

115-
function sh(binary, args) {
62+
function sh(binary, ...args) {
11663
const cmd = `${binary} ${args.join(" ")}`;
11764
if (GITHUB_ACTIONS_OUTPUT) {
11865
core.startGroup(binary);
@@ -128,21 +75,19 @@ function sh(binary, args) {
12875
}
12976
} finally {
13077
if (GITHUB_ACTIONS_OUTPUT)
131-
core.endGroup()
78+
core.endGroup();
13279
}
13380
}
13481

13582
async function runTests() {
136-
const shellBinary = logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup);
83+
const shellBinary = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup);
13784
let success = true;
138-
success &&= runTest("Run UnitTests", () => sh(shellBinary, [UNIT_TEST_PATH]));
139-
success &&= runTest("Run Complete Suite", () => sh(shellBinary, convertCliArgs(CLI_PATH)));
140-
success &&= runTest("Run Single Suite", () => {
141-
sh(shellBinary, convertCliArgs(CLI_PATH, "proxy-mobx"));
142-
});
143-
if (!success) {
144-
process.exit(1)
145-
}
85+
success &&= await runTest("Run UnitTests", () => sh(shellBinary, UNIT_TEST_PATH));
86+
success &&= await runCLITest("Run Single Suite", shellBinary, "proxy-mobx");
87+
success &&= await runCLITest("Run Disabled Suite", shellBinary, "disabled");
88+
success &&= await runCLITest("Run Default Suite", shellBinary);
89+
if (!success)
90+
process.exit(1);
14691
}
14792

14893
function jsvuOSName() {
@@ -161,30 +106,24 @@ function jsvuOSName() {
161106
default: throw new Error("Unsupported architecture");
162107
}
163108
};
164-
return `${osName()}${osArch()}`
109+
return `${osName()}${osArch()}`;
165110
}
166111

167112
const DEFAULT_JSC_LOCATION = "/System/Library/Frameworks/JavaScriptCore.framework/Versions/Current/Helpers/jsc"
168113

169114
function testSetup() {
170-
sh("jsvu", [`--engines=${SHELL_NAME}`, `--os=${jsvuOSName()}`]);
115+
sh("jsvu", `--engines=${SHELL_NAME}`, `--os=${jsvuOSName()}`);
171116
let shellBinary = path.join(os.homedir(), ".jsvu/bin", SHELL_NAME);
172117
if (!fs.existsSync(shellBinary) && SHELL_NAME == "javascriptcore")
173-
shellBinary = DEFAULT_JSC_LOCATION
118+
shellBinary = DEFAULT_JSC_LOCATION;
174119
if (!fs.existsSync(shellBinary))
175120
throw new Error(`Could not find shell binary: ${shellBinary}`);
176-
log(`Installed JavaScript Shell: ${shellBinary}`);
177-
return shellBinary
121+
logInfo(`Installed JavaScript Shell: ${shellBinary}`);
122+
return shellBinary;
178123
}
179124

180-
function runTest(testName, test) {
181-
try {
182-
logGroup(testName, test)
183-
} catch(e) {
184-
logError("TEST FAILED")
185-
return false
186-
}
187-
return true
125+
function runCLITest(name, shellBinary, ...args) {
126+
return runTest(name, () => sh(shellBinary, ...convertCliArgs(CLI_PATH, ...args)));
188127
}
189128

190129
setImmediate(runTests);

0 commit comments

Comments
 (0)