Skip to content

Commit f4208ff

Browse files
henrikingohenrikingo
committed
Add time CLI utility to supported benchmark tools
1 parent fd128cf commit f4208ff

11 files changed

Lines changed: 259 additions & 1 deletion

File tree

‎.github/workflows/time.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: Time (CLI utility) Example
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
branches:
9+
- master
10+
11+
permissions:
12+
contents: write
13+
deployments: write
14+
15+
jobs:
16+
benchmark:
17+
name: Run Some silly benchmark under `time`
18+
runs-on: ubuntu-latest
19+
steps:
20+
- uses: actions/checkout@v3
21+
- name: Run some silly benchmark with time
22+
run: (time (head --bytes=1000000000 /dev/random >/dev/null)) 2>&1 | tee output.txt
23+
24+
- name: Store benchmark result
25+
uses: benchmark-action/github-action-benchmark@v1
26+
with:
27+
name: Time Benchmark
28+
tool: 'time'
29+
output-file-path: output.txt
30+
github-token: ${{ secrets.GITHUB_TOKEN }}
31+
auto-push: true
32+
# Show alert with commit comment on detecting possible performance regression
33+
alert-threshold: '200%'
34+
comment-on-alert: true
35+
fail-on-alert: true
36+
alert-comment-cc-users: '@ktrz','@henrikingo'
37+
38+
- name: Store benchmark result - separate results repo
39+
uses: benchmark-action/github-action-benchmark@v1
40+
with:
41+
name: Time Benchmark
42+
tool: 'time'
43+
output-file-path: output.txt
44+
github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }}
45+
auto-push: true
46+
# Show alert with commit comment on detecting possible performance regression
47+
alert-threshold: '200%'
48+
comment-on-alert: true
49+
fail-on-alert: true
50+
alert-comment-cc-users: '@ktrz','@henrikingo'
51+
gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ This action currently supports the following tools:
2626
- [Benchmark.Net][benchmarkdotnet] for .Net projects
2727
- [benchmarkluau](https://github.com/Roblox/luau/tree/master/bench) for Luau projects
2828
- [JMH][jmh] for Java projects
29+
- [time][time-cli-util] CLI utility to measure runtime time of any Linux executable
2930
- Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter'
3031

3132
Multiple languages in the same repository are supported for polyglot projects.
@@ -51,6 +52,7 @@ definitions are in [.github/workflows/](./.github/workflows) directory. Live wor
5152
| .Net | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) |
5253
| Java | [![Java Example Workflow][java-badge]][java-workflow-example] | [examples/java](./examples/java) |
5354
| Luau | Coming soon | Coming soon |
55+
| Time (CLI utility) | [`time`][time-workflow-example]
5456

5557
All benchmark charts from above workflows are gathered in GitHub pages:
5658

@@ -651,6 +653,7 @@ Every release will appear on your GitHub notifications page.
651653
[examples-page]: https://benchmark-action.github.io/github-action-benchmark/dev/bench/
652654
[pytest-benchmark]: https://pypi.org/project/pytest-benchmark/
653655
[pytest]: https://pypi.org/project/pytest/
656+
[time-cli-util]: https://man7.org/linux/man-pages/man1/time.1.html
654657
[alert-comment-example]: https://github.com/benchmark-action/github-action-benchmark/commit/077dde1c236baba9244caad4d9e82ea8399dae20#commitcomment-36047186
655658
[rust-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Rust+Example%22
656659
[go-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Go+Example%22
@@ -660,6 +663,7 @@ Every release will appear on your GitHub notifications page.
660663
[catch2-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22
661664
[julia-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Julia+Example+with+BenchmarkTools.jl%22
662665
[java-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22JMH+Example%22
666+
[time-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22time+Example%22
663667
[help-watch-release]: https://docs.github.com/en/github/receiving-notifications-about-activity-on-github/watching-and-unwatching-releases-for-a-repository
664668
[help-github-token]: https://docs.github.com/en/actions/security-guides/automatic-token-authentication
665669
[minimal-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Example+for+minimal+setup%22

‎action-types.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ inputs:
1414
- jmh
1515
- benchmarkdotnet
1616
- benchmarkluau
17+
- time
1718
- customBiggerIsBetter
1819
- customSmallerIsBetter
1920
output-file-path:

‎action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ inputs:
1111
required: true
1212
default: 'Benchmark'
1313
tool:
14-
description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs", "pytest", "googlecpp", "catch2", "julia", "benchmarkdotnet", "customBiggerIsBetter", "customSmallerIsBetter"'
14+
description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs", "pytest", "googlecpp", "catch2", "julia", "benchmarkdotnet", "time", "customBiggerIsBetter", "customSmallerIsBetter"'
1515
required: true
1616
output-file-path:
1717
description: 'A path to file which contains the benchmark output'

‎examples/time/README.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
Example for parsing output of the `time` CLI utility
2+
====================================================
3+
4+
- [Workflow for this example](../../.github/workflows/time.yml)
5+
- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Rust+Example%22)
6+
- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/)
7+
8+
This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark)
9+
with [`cargo bench`](https://doc.rust-lang.org/cargo/commands/cargo-bench.html).
10+
11+
## Run benchmarks
12+
13+
Official documentation for usage of `time`:
14+
15+
https://man7.org/linux/man-pages/man1/time.1.html
16+
17+
So to run a benchmark, simply call time first:
18+
19+
```yaml
20+
- name: Measure execution time with `time`
21+
run: (time (head --bytes=1000000000 /dev/random >/dev/null)) 2>&1 | tee output.txt
22+
```
23+
24+
Note the `2>&1` too capture also stderr output. By default that's where `time` will direct its output.
25+
26+
27+
## Process benchmark results
28+
29+
Store the benchmark results with step using the action. Please set `cargo` to `tool` input.
30+
31+
```yaml
32+
- name: Store benchmark result
33+
uses: benchmark-action/github-action-benchmark@v1
34+
with:
35+
tool: 'time'
36+
output-file-path: output.txt
37+
```
38+
39+
Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage.
40+
41+

‎src/config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ export const VALID_TOOLS = [
3838
'julia',
3939
'jmh',
4040
'benchmarkdotnet',
41+
'time',
4142
'customBiggerIsBetter',
4243
'customSmallerIsBetter',
4344
] as const;

‎src/extract.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import { promises as fs } from 'fs';
33
import * as github from '@actions/github';
44
import { Config, ToolType } from './config';
5+
import * as core from '@actions/core';
56

67
export interface BenchmarkResult {
78
name: string;
@@ -690,6 +691,93 @@ function extractLuauBenchmarkResult(output: string): BenchmarkResult[] {
690691
return results;
691692
}
692693

694+
function parseTimeOutput(line: string): number | undefined {
695+
core.debug(`parse time ${line}`);
696+
const t = line.split('\t')[1];
697+
if (t === undefined) return;
698+
const tparts = t.split('m');
699+
700+
core.debug(tparts[0]);
701+
if (tparts[1] === undefined || !tparts[1].endsWith('s')) return;
702+
return parseFloat(tparts[0]) * 60.0 + parseFloat(tparts[1].substring(-1));
703+
}
704+
/* Expect to process text with the following structure:
705+
706+
$ time echo Hello there
707+
Hello there
708+
709+
real 0m0,100s
710+
user 0m0,020s
711+
sys 0m0,003s
712+
$ time echo Hello there
713+
Hello again
714+
715+
real 0m0,100s
716+
user 0m0,020s
717+
sys 0m0,003s
718+
719+
Into:
720+
[{"name": "real",
721+
"value": 0,1,
722+
"unit": "s",
723+
"extra": "testName: Hello there"},
724+
...
725+
*/
726+
function extractTimeBenchmarkResult(output: string): BenchmarkResult[] {
727+
const lines = output.split(/\n/);
728+
const results: BenchmarkResult[] = [];
729+
let firstline = true;
730+
let name: string | undefined = undefined;
731+
732+
for (const line of lines) {
733+
core.debug(line);
734+
if (firstline) {
735+
name = line;
736+
firstline = false;
737+
core.debug(`firstline: ${name}`);
738+
continue;
739+
}
740+
if (line.startsWith('real')) {
741+
const v = parseTimeOutput(line);
742+
core.debug(`v: ${v}`);
743+
if (v !== undefined)
744+
results.push({
745+
extra: name,
746+
name: 'real',
747+
value: v,
748+
unit: 's',
749+
});
750+
}
751+
if (line.startsWith('user')) {
752+
const v = parseTimeOutput(line);
753+
754+
core.debug(`v: ${v}`);
755+
if (v !== undefined)
756+
results.push({
757+
extra: name,
758+
name: 'user',
759+
value: v,
760+
unit: 's',
761+
});
762+
}
763+
if (line.startsWith('sys')) {
764+
const v = parseTimeOutput(line);
765+
core.debug(`v: ${v}`);
766+
if (v !== undefined)
767+
results.push({
768+
extra: name,
769+
name: 'sys',
770+
value: v,
771+
unit: 's',
772+
});
773+
firstline = true;
774+
}
775+
core.debug('loop');
776+
}
777+
778+
return results;
779+
}
780+
693781
export async function extractResult(config: Config): Promise<Benchmark> {
694782
const output = await fs.readFile(config.outputFilePath, 'utf8');
695783
const { tool, githubToken, ref } = config;
@@ -723,6 +811,9 @@ export async function extractResult(config: Config): Promise<Benchmark> {
723811
case 'benchmarkdotnet':
724812
benches = extractBenchmarkDotnetResult(output);
725813
break;
814+
case 'time':
815+
benches = extractTimeBenchmarkResult(output);
816+
break;
726817
case 'customBiggerIsBetter':
727818
benches = extractCustomBenchmarkResult(output);
728819
break;

‎src/write.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ function biggerIsBetter(tool: ToolType): boolean {
8181
return false;
8282
case 'benchmarkdotnet':
8383
return false;
84+
case 'time':
85+
return false;
8486
case 'customBiggerIsBetter':
8587
return true;
8688
case 'customSmallerIsBetter':

‎test/__snapshots__/extract.spec.ts.snap‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -968,3 +968,56 @@ rounds: 5",
968968
"tool": "pytest",
969969
}
970970
`;
971+
972+
exports[`extractResult() extracts benchmark output from time - time_output.txt 1`] = `
973+
{
974+
"benches": [
975+
{
976+
"extra": "Hello there",
977+
"name": "real",
978+
"unit": "s",
979+
"value": 0.1,
980+
},
981+
{
982+
"extra": "Hello there",
983+
"name": "user",
984+
"unit": "s",
985+
"value": 0.02,
986+
},
987+
{
988+
"extra": "Hello there",
989+
"name": "sys",
990+
"unit": "s",
991+
"value": 0.003,
992+
},
993+
{
994+
"extra": "Hello again",
995+
"name": "real",
996+
"unit": "s",
997+
"value": 0.1,
998+
},
999+
{
1000+
"extra": "Hello again",
1001+
"name": "user",
1002+
"unit": "s",
1003+
"value": 0.02,
1004+
},
1005+
{
1006+
"extra": "Hello again",
1007+
"name": "sys",
1008+
"unit": "s",
1009+
"value": 0.003,
1010+
},
1011+
],
1012+
"commit": {
1013+
"author": null,
1014+
"committer": null,
1015+
"id": "123456789abcdef",
1016+
"message": "this is dummy",
1017+
"timestamp": "dummy timestamp",
1018+
"url": "https://github.com/dummy/repo",
1019+
},
1020+
"date": 1712131503296,
1021+
"tool": "time",
1022+
}
1023+
`;

‎test/data/extract/time_output.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
Hello there
2+
3+
real 0m0.100s
4+
user 0m0.020s
5+
sys 0m0.003s
6+
Hello again
7+
8+
real 0m0.100s
9+
user 0m0.020s
10+
sys 0m0.003s

0 commit comments

Comments
 (0)