ci: Add internal benchmark tool

Author: matz3

internal/benchmark/README.md

@@ -0,0 +1,110 @@
+# UI5 CLI Benchmark Tool
+
+A benchmarking utility for measuring and comparing the performance of UI5 CLI commands across different git revisions.
+
+## Prerequisites
+
+This tool requires [hyperfine](https://github.com/sharkdp/hyperfine) to be installed. Follow the installation instructions in the hyperfine repository for your platform.
+
+## Installation
+
+Make the `ui5-cli-benchmark` binary available globally:
+
+```bash
+npm link
+```
+
+## Usage
+
+```bash
+ui5-cli-benchmark run <path-to-config> [<project-dir>...]
+```
+
+### Arguments
+
+- `<path-to-config>` - Path to a YAML configuration file (required)
+- `[<project-dir>...]` - One or more project directories to benchmark (optional, defaults to current working directory)
+
+### Example
+
+```bash
+# Run benchmarks using example config in current directory
+ui5-cli-benchmark run config/.example.yaml
+
+# Run benchmarks in specific project directories
+ui5-cli-benchmark run config/.example.yaml /path/to/project1 /path/to/project2
+```
+
+## Configuration
+
+Create a YAML configuration file with the following structure:
+
+### Revisions
+
+Define the git revisions to benchmark:
+
+```yaml
+revisions:
+  baseline:
+    name: "Baseline"
+    revision:
+      merge_base_from: "feat/example-feature"
+      target_branch: "main"
+  example_feature:
+    name: "Example Feature"
+    revision: "feat/example-feature"
+```
+
+Each revision can specify:
+- `name` - Display name for the revision
+- `revision` - Either a branch/commit hash or an object with `merge_base_from` and `target_branch` to compute the merge base
+
+### Hyperfine Settings
+
+Configure the benchmark runner (uses [hyperfine](https://github.com/sharkdp/hyperfine)):
+
+```yaml
+hyperfine:
+  warmup: 1  # Number of warmup runs
+  runs: 10   # Number of benchmark runs
+```
+
+### Groups
+
+Define logical groups for organizing benchmark results:
+
+```yaml
+groups:
+  build:
+    name: "ui5 build"
+```
+
+### Benchmarks
+
+Define the commands to benchmark:
+
+```yaml
+benchmarks:
+  - command: "build"
+    prepare: "rm -rf .ui5-cache"  # Optional: command to run before each benchmark
+    groups:
+      build:
+        name: "build"
+    revisions:  # Optional: limit to specific revisions
+      - "example_feature"
+```
+
+Each benchmark can specify:
+- `command` - The UI5 CLI command to run (e.g., "build", "build --clean-dest")
+- `prepare` - Optional shell command to run before each benchmark iteration
+- `groups` - Group(s) this benchmark belongs to with display names
+- `revisions` - Optional array to limit which revisions run this benchmark (defaults to all)
+
+## Output
+
+The tool generates:
+- Console output with progress and summary
+- Markdown report with benchmark results
+- JSON report with raw data
+
+Results are organized by revision and group for easy comparison.

internal/benchmark/cli.js

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+
+import {fileURLToPath} from "node:url";
+import path from "node:path";
+import fs from "node:fs";
+import BenchmarkRunner from "./lib/BenchmarkRunner.js";
+import git from "./lib/utils/git.js";
+import npm from "./lib/utils/npm.js";
+import {spawnProcess} from "./lib/utils/process.js";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+function printUsageAndExit() {
+	console.error(
+		"Usage:\n\t" +
+		"ui5-cli-benchmark run <path-to-config> [<project-dir>...]"
+	);
+	process.exit(1);
+}
+
+export const commands = {
+	async run(args, options = {}) {
+		const configFilePath = args[0];
+		const projectDirs = args.slice(1);
+
+		// Validate arguments
+		if (!configFilePath) {
+			return printUsageAndExit();
+		}
+
+		// Determine repository and CLI paths
+		const repositoryPath = path.resolve(__dirname, "../..");
+		const ui5CliPath = path.resolve(repositoryPath, "packages/cli/bin/ui5.cjs");
+
+		// Create BenchmarkRunner with injected dependencies
+		const benchmarkRunner = new BenchmarkRunner({
+			git: options.git || git,
+			npm: options.npm || npm,
+			spawnProcess: options.spawnProcess || spawnProcess,
+			fs: options.fs || fs
+		});
+
+		// Run benchmarks
+		const result = await benchmarkRunner.run({
+			configFilePath,
+			repositoryPath,
+			ui5CliPath,
+			projectDirs: projectDirs.length > 0 ? projectDirs : undefined,
+			timestamp: options.timestamp
+		});
+
+		if (!result.success) {
+			process.exit(1);
+		}
+	}
+};
+
+async function main() {
+	const args = process.argv.slice(2);
+
+	if (args.length === 0 || args[0] === "-h" || args[0] === "--help") {
+		return printUsageAndExit();
+	}
+
+	const command = args[0];
+	const commandArgs = args.slice(1);
+	const fn = commands[command];
+
+	// Validate command name
+	if (!fn) {
+		process.stderr.write(`Unknown command: '${command}'\n\n`);
+		return process.exit(1);
+	}
+
+	// Execute handler
+	try {
+		await fn(commandArgs);
+	} catch (error) {
+		console.error(`Unexpected error: ${error.message}`);
+		console.error("Stack trace:", error.stack);
+
+		process.exit(1);
+	}
+}
+
+
+// Handle uncaught exceptions
+process.on("uncaughtException", (error) => {
+	console.error("Uncaught exception:", error.message);
+	process.exit(1);
+});
+
+process.on("unhandledRejection", (reason, promise) => {
+	console.error("Unhandled rejection at:", promise, "reason:", reason);
+	process.exit(1);
+});
+
+main();

internal/benchmark/config/.example.yaml

@@ -0,0 +1,35 @@
+# Example Benchmark configuration for UI5 CLI
+
+revisions:
+  baseline:
+    name: "Baseline"
+    revision:
+      merge_base_from: "feat/example-feature"
+      target_branch: "main"
+  example_feature:
+    name: "Example Feature"
+    revision: "feat/example-feature"
+
+hyperfine:
+  warmup: 1
+  runs: 10
+
+groups:
+  build:
+    name: "ui5 build"
+
+benchmarks:
+
+  - command: "build"
+    groups:
+      build:
+        name: "build"
+
+  - command: "build --some-new-flag"
+    groups:
+      build:
+        name: "build (with some new flag)"
+    revisions:
+      # Benchmark with some new flag is only relevant for the feature revision,
+      # as the baseline does not contain the flag
+      - "example_feature"

internal/benchmark/eslint.config.js

@@ -0,0 +1,10 @@
+import commonConfig from "../../eslint.common.config.js";
+
+export default [
+	...commonConfig,
+	{
+		rules: {
+			"no-console": "off" // Allow console output in CLI tools
+		}
+	}
+];