Skip to content

Commit 484083c

Browse files
jdrhynejdrhyne
committed
Merge task/t7-benchmark
2 parents 84cae5a + c599b3a commit 484083c

2 files changed

Lines changed: 245 additions & 6 deletions

File tree

‎src/commands/benchmark.ts‎

Lines changed: 109 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,117 @@
11
import { Command } from "commander";
2-
import { notImplemented } from "./notImplemented";
2+
import fs from "node:fs/promises";
3+
import path from "node:path";
4+
5+
type BenchmarkOptions = {
6+
suite?: string;
7+
out?: string;
8+
};
9+
10+
type BenchmarkRun = {
11+
name: string;
12+
startedAt: string;
13+
finishedAt: string;
14+
metrics: {
15+
latencyMs: number;
16+
throughputRps: number;
17+
errorRate: number;
18+
};
19+
};
20+
21+
const toErrorMessage = (error: unknown): string =>
22+
error instanceof Error ? error.message : String(error);
23+
24+
const resolvePath = (value: string): string =>
25+
path.isAbsolute(value) ? value : path.resolve(process.cwd(), value);
26+
27+
const normalizeOption = (value: string | undefined): string | undefined => {
28+
if (typeof value !== "string") {
29+
return undefined;
30+
}
31+
const trimmed = value.trim();
32+
return trimmed ? trimmed : undefined;
33+
};
34+
35+
const ensureOutputPath = async (filePath: string): Promise<void> => {
36+
try {
37+
const stat = await fs.stat(filePath);
38+
if (stat.isDirectory()) {
39+
throw new Error("Output path must be a file.");
40+
}
41+
} catch (error) {
42+
if (
43+
error instanceof Error &&
44+
"code" in error &&
45+
(error as NodeJS.ErrnoException).code === "ENOENT"
46+
) {
47+
await fs.mkdir(path.dirname(filePath), { recursive: true });
48+
return;
49+
}
50+
throw error;
51+
}
52+
await fs.mkdir(path.dirname(filePath), { recursive: true });
53+
};
54+
55+
const buildPlaceholderRuns = (startedAt: Date): BenchmarkRun[] => {
56+
const baseLatency = 140;
57+
const runs: BenchmarkRun[] = [];
58+
for (let index = 0; index < 3; index += 1) {
59+
const runStart = new Date(startedAt.getTime() + index * 1000);
60+
const runEnd = new Date(runStart.getTime() + 450 + index * 40);
61+
runs.push({
62+
name: `run-${index + 1}`,
63+
startedAt: runStart.toISOString(),
64+
finishedAt: runEnd.toISOString(),
65+
metrics: {
66+
latencyMs: baseLatency + index * 12,
67+
throughputRps: 4.2 + index * 0.3,
68+
errorRate: 0.001 + index * 0.0005,
69+
},
70+
});
71+
}
72+
return runs;
73+
};
374

475
export const registerBenchmarkCommand = (program: Command) => {
576
program
677
.command("benchmark")
778
.description("Benchmark suite")
8-
.action(() => {
9-
notImplemented("benchmark");
79+
.option("--suite <name>", "Benchmark suite name", "W1-W6")
80+
.option("--out <path>", "Path to the output results JSON", "results.json")
81+
.action(async (options: BenchmarkOptions = {}) => {
82+
try {
83+
const suite = normalizeOption(options.suite) ?? "W1-W6";
84+
const outputValue = normalizeOption(options.out) ?? "results.json";
85+
const resolvedOutput = resolvePath(outputValue);
86+
87+
await ensureOutputPath(resolvedOutput);
88+
89+
const startedAt = new Date();
90+
const runs = buildPlaceholderRuns(startedAt);
91+
const finishedAt = new Date(
92+
new Date(runs[runs.length - 1]?.finishedAt ?? startedAt.toISOString()).getTime()
93+
);
94+
const avgLatencyMs =
95+
runs.reduce((sum, run) => sum + run.metrics.latencyMs, 0) / runs.length;
96+
97+
const result = {
98+
status: "stub",
99+
suite,
100+
startedAt: startedAt.toISOString(),
101+
finishedAt: finishedAt.toISOString(),
102+
runs,
103+
summary: {
104+
runCount: runs.length,
105+
avgLatencyMs,
106+
},
107+
warnings: ["Benchmark results are placeholders."],
108+
};
109+
110+
await fs.writeFile(resolvedOutput, JSON.stringify(result, null, 2), "utf8");
111+
console.log(`Wrote benchmark results to ${resolvedOutput}`);
112+
} catch (error) {
113+
console.error(`Benchmark failed: ${toErrorMessage(error)}`);
114+
process.exit(1);
115+
}
10116
});
11117
};

‎src/commands/report.ts‎

Lines changed: 136 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,144 @@
11
import { Command } from "commander";
2-
import { notImplemented } from "./notImplemented";
2+
import fs from "node:fs/promises";
3+
import path from "node:path";
4+
5+
type ReportOptions = {
6+
in?: string;
7+
out?: string;
8+
};
9+
10+
type BenchmarkResults = {
11+
status?: string;
12+
suite?: string;
13+
runs?: Array<{
14+
metrics?: {
15+
latencyMs?: number;
16+
};
17+
}>;
18+
summary?: {
19+
avgLatencyMs?: number;
20+
};
21+
warnings?: string[];
22+
};
23+
24+
const toErrorMessage = (error: unknown): string =>
25+
error instanceof Error ? error.message : String(error);
26+
27+
const resolvePath = (value: string): string =>
28+
path.isAbsolute(value) ? value : path.resolve(process.cwd(), value);
29+
30+
const normalizeOption = (value: string | undefined): string | undefined => {
31+
if (typeof value !== "string") {
32+
return undefined;
33+
}
34+
const trimmed = value.trim();
35+
return trimmed ? trimmed : undefined;
36+
};
37+
38+
const ensureInputFile = async (filePath: string): Promise<void> => {
39+
let stat;
40+
try {
41+
stat = await fs.stat(filePath);
42+
} catch (error) {
43+
throw new Error(`Unable to access input file: ${toErrorMessage(error)}`);
44+
}
45+
46+
if (!stat.isFile()) {
47+
throw new Error("Input path must point to a file.");
48+
}
49+
};
50+
51+
const ensureOutputPath = async (filePath: string): Promise<void> => {
52+
try {
53+
const stat = await fs.stat(filePath);
54+
if (stat.isDirectory()) {
55+
throw new Error("Output path must be a file.");
56+
}
57+
} catch (error) {
58+
if (
59+
error instanceof Error &&
60+
"code" in error &&
61+
(error as NodeJS.ErrnoException).code === "ENOENT"
62+
) {
63+
await fs.mkdir(path.dirname(filePath), { recursive: true });
64+
return;
65+
}
66+
throw error;
67+
}
68+
await fs.mkdir(path.dirname(filePath), { recursive: true });
69+
};
70+
71+
const isFiniteNumber = (value: unknown): value is number =>
72+
typeof value === "number" && Number.isFinite(value);
373

474
export const registerReportCommand = (program: Command) => {
575
program
676
.command("report")
777
.description("Reporting tools")
8-
.action(() => {
9-
notImplemented("report");
78+
.option("--in <path>", "Path to the benchmark results JSON", "results.json")
79+
.option("--out <path>", "Path to the output report", "report.md")
80+
.action(async (options: ReportOptions = {}) => {
81+
try {
82+
const inputValue = normalizeOption(options.in) ?? "results.json";
83+
const outputValue = normalizeOption(options.out) ?? "report.md";
84+
const resolvedInput = resolvePath(inputValue);
85+
const resolvedOutput = resolvePath(outputValue);
86+
87+
await ensureInputFile(resolvedInput);
88+
await ensureOutputPath(resolvedOutput);
89+
90+
const raw = await fs.readFile(resolvedInput, "utf8");
91+
let parsed: BenchmarkResults;
92+
try {
93+
parsed = JSON.parse(raw) as BenchmarkResults;
94+
} catch (error) {
95+
throw new Error(`Unable to parse JSON: ${toErrorMessage(error)}`);
96+
}
97+
98+
const suite =
99+
typeof parsed.suite === "string" && parsed.suite.trim()
100+
? parsed.suite.trim()
101+
: "Unknown suite";
102+
const runs = Array.isArray(parsed.runs) ? parsed.runs : [];
103+
const runCount = runs.length;
104+
105+
const latencies = runs
106+
.map((run) => run?.metrics?.latencyMs)
107+
.filter(isFiniteNumber);
108+
const avgLatency =
109+
latencies.length > 0
110+
? latencies.reduce((sum, value) => sum + value, 0) / latencies.length
111+
: isFiniteNumber(parsed.summary?.avgLatencyMs)
112+
? parsed.summary?.avgLatencyMs
113+
: undefined;
114+
const avgLatencyText =
115+
typeof avgLatency === "number" ? avgLatency.toFixed(2) : "N/A";
116+
117+
const status = parsed.status ? parsed.status : "unknown";
118+
const warnings = Array.isArray(parsed.warnings) ? parsed.warnings : [];
119+
120+
const lines = [
121+
"# Benchmark Report",
122+
"",
123+
`- Status: ${status}`,
124+
`- Suite: ${suite}`,
125+
`- Runs: ${runCount}`,
126+
`- Avg latency (ms): ${avgLatencyText}`,
127+
"",
128+
warnings.length > 0 ? "## Warnings" : "",
129+
...warnings.map((warning) => `- ${warning}`),
130+
warnings.length > 0 ? "" : "",
131+
"## Notes",
132+
"- If this report is marked as stub, replace placeholder metrics with real benchmark output.",
133+
"- Re-run `dws benchmark --suite <name>` after integrating the DWS API.",
134+
"",
135+
].filter((line) => line !== "");
136+
137+
await fs.writeFile(resolvedOutput, lines.join("\n"), "utf8");
138+
console.log(`Wrote report to ${resolvedOutput}`);
139+
} catch (error) {
140+
console.error(`Report failed: ${toErrorMessage(error)}`);
141+
process.exit(1);
142+
}
10143
});
11144
};

0 commit comments

Comments
 (0)