Skip to content

Commit 04af5f7

Browse files
felsenhowerfelsenhower
committed
Add benchmarks
1 parent e69924e commit 04af5f7

6 files changed

Lines changed: 240 additions & 0 deletions

File tree

‎benchmark/.python-version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.13

‎benchmark/benchmark.sh‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
IFS=$'\n\t'
5+
6+
[ "${SHELLCHECK:-0}" == "1" ] && shellcheck "$0"
7+
8+
if (( "$#" < 1 )) ; then
9+
echo 'Error: Must supply path to reference implementation!'
10+
exit 1
11+
fi
12+
13+
REFERENCE="$1"
14+
15+
GAUSS_SEIDEL_ARGS=(1 1 100 2 2 100)
16+
JACOBI_ARGS=(1 2 100 2 2 100)
17+
18+
VARIANTS=(simple numba np_vectorize)
19+
20+
declare -A PYTHON_VERSIONS=(
21+
[simple]='cpython3.13'
22+
[numba]='cpython3.10'
23+
[np_vectorize]='cpython3.13'
24+
)
25+
26+
OUTPUT_FILE="$(realpath benchmark_results.csv)"
27+
28+
cd ..
29+
30+
export LANG=C
31+
export LC_NUMERIC=C
32+
33+
function format_runtime {
34+
awk '
35+
/Calculation time/{printf(" %-16.2f,", $3);};
36+
/real/{printf(" %-13.2f\n", $2);};
37+
'
38+
}
39+
40+
{
41+
printf '%-12s, %s, %-11s, %-16s, %-13s\n' variant i method runtime_internal runtime_total
42+
43+
for i in {1..3} ; do
44+
printf '%-12s, %s, %-11s,' 'reference' "$i" 'Gauß-Seidel'
45+
{ time -p "${REFERENCE}" "${GAUSS_SEIDEL_ARGS[@]}" ; } 2>&1 | format_runtime
46+
printf '%-12s, %s, %-11s,' 'reference' "$i" 'Jacobi'
47+
{ time -p "${REFERENCE}" "${JACOBI_ARGS[@]}" ; } 2>&1 | format_runtime
48+
done
49+
50+
for variant in "${VARIANTS[@]}" ; do
51+
pushd "$variant" > /dev/null
52+
python="${PYTHON_VERSIONS["$variant"]}"
53+
# i=0 is added here to warm up the JIT for the numba version, if needed.
54+
for i in {0..3} ; do
55+
printf '%-12s, %s, %-11s,' "$variant" "$i" 'Gauß-Seidel'
56+
{ time -p uv run --python "$python" main.py "${GAUSS_SEIDEL_ARGS[@]}" ; } 2>&1 | format_runtime
57+
printf '%-12s, %s, %-11s,' "$variant" "$i" 'Jacobi'
58+
{ time -p uv run --python "$python" main.py "${JACOBI_ARGS[@]}" ; } 2>&1 | format_runtime
59+
done
60+
popd > /dev/null
61+
done
62+
} | tee "$OUTPUT_FILE"

‎benchmark/benchmark_results.csv‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
variant , i, method , runtime_internal, runtime_total
2+
reference , 1, Gauß-Seidel, 0.55 , 0.55
3+
reference , 1, Jacobi , 0.48 , 0.48
4+
reference , 2, Gauß-Seidel, 0.59 , 0.60
5+
reference , 2, Jacobi , 0.48 , 0.48
6+
reference , 3, Gauß-Seidel, 0.55 , 0.55
7+
reference , 3, Jacobi , 0.51 , 0.52
8+
simple , 0, Gauß-Seidel, 52.43 , 52.72
9+
simple , 0, Jacobi , 52.42 , 52.72
10+
simple , 1, Gauß-Seidel, 52.12 , 52.41
11+
simple , 1, Jacobi , 52.67 , 52.97
12+
simple , 2, Gauß-Seidel, 51.59 , 51.88
13+
simple , 2, Jacobi , 51.71 , 51.00
14+
simple , 3, Gauß-Seidel, 51.74 , 52.03
15+
simple , 3, Jacobi , 52.48 , 52.78
16+
numba , 0, Gauß-Seidel, 0.69 , 1.12
17+
numba , 0, Jacobi , 0.41 , 0.84
18+
numba , 1, Gauß-Seidel, 0.69 , 1.14
19+
numba , 1, Jacobi , 0.42 , 0.86
20+
numba , 2, Gauß-Seidel, 0.69 , 1.14
21+
numba , 2, Jacobi , 0.41 , 0.85
22+
numba , 3, Gauß-Seidel, 0.73 , 1.17
23+
numba , 3, Jacobi , 0.42 , 0.87
24+
np_vectorize, 0, Gauß-Seidel, 51.49 , 51.77
25+
np_vectorize, 0, Jacobi , 0.21 , 0.50
26+
np_vectorize, 1, Gauß-Seidel, 55.31 , 55.60
27+
np_vectorize, 1, Jacobi , 0.21 , 0.49
28+
np_vectorize, 2, Gauß-Seidel, 55.39 , 55.68
29+
np_vectorize, 2, Jacobi , 0.21 , 0.49
30+
np_vectorize, 3, Gauß-Seidel, 54.83 , 55.12
31+
np_vectorize, 3, Jacobi , 0.22 , 0.51

‎benchmark/main.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
import polars as pl
2+
from io import StringIO
3+
from uncertainties import ufloat
4+
5+
def preprocess_file() -> StringIO:
6+
sio = StringIO()
7+
with open("benchmark_results.csv") as f:
8+
for line in f:
9+
line = line.strip()
10+
line = line.split(",")
11+
line = [f.strip() for f in line]
12+
line = ",".join(line)
13+
sio.write(f"{line}\n")
14+
sio.seek(0)
15+
return sio
16+
17+
def extract_baseline_runtime(df, method: str, col_name: str):
18+
return ufloat(*(df.filter(pl.col("variant") == "reference", pl.col("method") == method).to_dicts()[0][col_name]))
19+
20+
def main():
21+
df = pl.read_csv(preprocess_file())
22+
df = df.filter(
23+
pl.col("i") > 0
24+
)
25+
df = (
26+
df
27+
.group_by(["variant", "method"], maintain_order=True)
28+
.agg(
29+
pl.col("runtime_internal").mean().alias("runtime_internal_mean"),
30+
pl.col("runtime_internal").std().alias("runtime_internal_std"),
31+
pl.col("runtime_total").mean().alias("runtime_total_mean"),
32+
pl.col("runtime_total").std().alias("runtime_total_std"),
33+
)
34+
)
35+
df = df.select(
36+
pl.col("variant"),
37+
pl.col("method"),
38+
pl.concat_list("runtime_internal_mean", "runtime_internal_std").alias("runtime_internal"),
39+
pl.concat_list("runtime_total_mean", "runtime_total_std").alias("runtime_total")
40+
)
41+
42+
def runtime_factor(method, runtime, time_type):
43+
res = ufloat(*runtime) * 100 / extract_baseline_runtime(df, method, time_type)
44+
return res.n # [res.n, res.s]
45+
46+
df = df.with_columns(
47+
48+
pl.struct(["method", "runtime_internal"]).map_elements(
49+
lambda x: runtime_factor(x["method"], x["runtime_internal"], "runtime_internal"),
50+
return_dtype=pl.Float64(),
51+
# return_dtype=pl.List(pl.Float64()),
52+
).alias("runtime_internal_factor"),
53+
54+
pl.struct(["method", "runtime_total"]).map_elements(
55+
lambda x: runtime_factor(x["method"], x["runtime_total"], "runtime_total"),
56+
return_dtype=pl.Float64(),
57+
# return_dtype=pl.List(pl.Float64()),
58+
).alias("runtime_total_factor"),
59+
60+
)
61+
62+
63+
def format_ufloat_runtime(x) -> str:
64+
x = ufloat(*x)
65+
return "({:.3f} ± {:.3f}) s".format(x.n, x.s)
66+
67+
df = df.with_columns(
68+
pl.col("runtime_internal").map_elements(lambda x: format_ufloat_runtime(x), return_dtype=pl.String()),
69+
pl.col("runtime_total").map_elements(lambda x: format_ufloat_runtime(x), return_dtype=pl.String()),
70+
pl.col("runtime_internal_factor").map_elements(lambda x: f"{x:.2f}%", return_dtype=pl.String()),
71+
pl.col("runtime_total_factor").map_elements(lambda x: f"{x:.2f}%", return_dtype=pl.String()),
72+
)
73+
74+
with pl.Config(
75+
tbl_formatting="MARKDOWN",
76+
tbl_hide_column_data_types=True,
77+
tbl_hide_dataframe_shape=True,
78+
):
79+
print(df)
80+
81+
82+
if __name__ == "__main__":
83+
main()

‎benchmark/pyproject.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[project]
2+
name = "process-benchmarks"
3+
version = "0.1.0"
4+
description = "Add your description here"
5+
readme = "README.md"
6+
requires-python = ">=3.13"
7+
dependencies = [
8+
"polars>=1.36.1",
9+
"uncertainties>=3.2.3",
10+
]

‎benchmark/uv.lock‎

Lines changed: 53 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)