Skip to content

Commit e339b80

Browse files
ingydotnetingydotnet
committed
Add benchmarking and profiling
Add bench2.sh for timing benchmarks and profile.sh for CPU/memory profiling via pprof. Both scripts build glj from a specified glojure commit, compile yamlstar via gloat, and run the binary with load-average gating. Includes diagnostic output for verifying codegen (ApplyN/FnFuncN) is active.
1 parent 2db06a5 commit e339b80

6 files changed

Lines changed: 451 additions & 0 deletions

File tree

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/.cache/
22
/note/
3+
/worktree/
34
/ToDo
45
CLAUDE.md
56
.cpcache/
@@ -10,6 +11,8 @@ META-INF/
1011
/release-*
1112
/*.clj
1213
/*.json
14+
/*.md
15+
/*.txt
1316
/*.yaml
1417
/*.ys
1518
/out*

‎bench/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/profiles/

‎bench/bench.clj‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
(ns bench2
2+
(:require [yamlstar.core :as yaml]))
3+
4+
(def inputs
5+
[["scalar" "hello"]
6+
["mapping" "foo: 42"]
7+
["nested" "root:\n child1:\n key: value\n child2:\n - item1\n - item2\n - item3"]
8+
["types" "string: hello\ninteger: 42\nfloat: 3.14\nbool: true\nnull_val: null"]])
9+
10+
(defn now-ns []
11+
(.UnixNano (time.Now)))
12+
13+
(defn fmt-time [ns]
14+
(let [ms (/ (double ns) 1000000.0)]
15+
(if (>= ms 1000)
16+
(fmt.Sprintf "%8.2f s " (/ ms 1000))
17+
(if (< ms 1)
18+
(fmt.Sprintf "%7.3f ms" ms)
19+
(fmt.Sprintf "%7.1f ms" ms)))))
20+
21+
(defn -main []
22+
;; Cold call
23+
(let [t0 (now-ns)]
24+
(yaml/load "warmup: true")
25+
(println (str "cold: " (fmt-time (- (now-ns) t0))
26+
" (first call, includes ns init)")))
27+
28+
(println)
29+
(println (fmt.Sprintf "%-12s %12s" "input" "time"))
30+
(println (apply str (repeat 26 "-")))
31+
32+
;; Single call per input
33+
(let [limit-str (os.Getenv "LIMIT")
34+
limit-ms (when (not= limit-str "")
35+
(parse-long limit-str))]
36+
(loop [remaining inputs]
37+
(when (seq remaining)
38+
(let [[label input] (first remaining)
39+
t0 (now-ns)
40+
_ (yaml/load input)
41+
elapsed-ns (- (now-ns) t0)
42+
elapsed-ms (/ (double elapsed-ns) 1000000.0)]
43+
(println (fmt.Sprintf "%-12s %s" label (fmt-time elapsed-ns)))
44+
(if (and (= label "scalar") limit-ms (> elapsed-ms limit-ms))
45+
(println (fmt.Sprintf "(skipping remaining — scalar exceeded %d ms limit)" limit-ms))
46+
(recur (rest remaining)))))))
47+
48+
;; Bulk test - adaptive reps
49+
(let [t0 (now-ns)
50+
_ (yaml/load "foo: 42")
51+
probe-ms (/ (double (- (now-ns) t0)) 1000000.0)
52+
reps (cond (< probe-ms 10) 100
53+
(< probe-ms 500) 10
54+
:else 3)
55+
t0 (now-ns)]
56+
(dotimes [_ reps]
57+
(yaml/load "foo: 42"))
58+
(let [total-ns (- (now-ns) t0)
59+
total-ms (/ (double total-ns) 1000000.0)
60+
per-ms (/ total-ms reps)]
61+
(println)
62+
(println (fmt.Sprintf "%d x 'foo: 42': %s total, %s/call"
63+
reps (fmt-time total-ns) (fmt-time (long (/ total-ns reps))))))))

‎bench/bench.sh‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/usr/bin/env bash
2+
3+
# Build a native Go benchmark binary from Clojure sources via gloat,
4+
# then run it. Usage: ./bench2.sh [glojure-commit]
5+
# Assumes deps already bootstrapped (run make build once).
6+
7+
set -euo pipefail
8+
9+
ROOT=$(cd "$(dirname "$0")/.." && pwd)
10+
GLOJURE=$ROOT/../glojure
11+
COMMIT=${1:-gloat}
12+
13+
echo "=== 0. Clone glojure @ $COMMIT ==="
14+
GLOJURE_BUILD=/tmp/glojure/$COMMIT
15+
rm -fr "$GLOJURE_BUILD"
16+
git clone -q "$GLOJURE/.git" "$GLOJURE_BUILD"
17+
git -C "$GLOJURE_BUILD" checkout -q "$COMMIT"
18+
19+
# Auto-detect local root and gloat installation
20+
GLOAT_HOME=$HOME/.local/share/gloat
21+
if [[ -d /tmp/yamlstar-local/go-1.26.0 ]]; then
22+
LOCAL=/tmp/yamlstar-local
23+
GLOAT_DIR=$LOCAL/cache/gloat-main
24+
elif [[ -d $ROOT/.cache/.local/go-1.26.0 ]]; then
25+
LOCAL=$ROOT/.cache/.local
26+
GLOAT_DIR=$LOCAL/cache/gloat-main
27+
elif [[ -d $GLOAT_HOME/.cache/.local/go-1.26.0 ]]; then
28+
LOCAL=$GLOAT_HOME/.cache/.local
29+
GLOAT_DIR=$GLOAT_HOME
30+
else
31+
echo "ERROR: Cannot find Go installation" >&2
32+
exit 1
33+
fi
34+
35+
GO=$LOCAL/go-1.26.0/bin/go
36+
GLOAT=$GLOAT_DIR/bin/gloat
37+
GLJ=$LOCAL/bin/glj
38+
39+
# Find gloat's own glj path (may differ from $GLJ).
40+
# Always query from GLOAT_HOME which has the Makefile with gloat-vars.
41+
GLOAT_GLJ=$(cd "$GLOAT_HOME" && make --quiet --no-print-directory gloat-vars 2>/dev/null \
42+
| grep ':GLJ ' | sed 's/.*:GLJ "\(.*\)"/\1/')
43+
44+
BUILD_DIR=/tmp/yamlstar-bench
45+
BINARY=$BUILD_DIR/bench
46+
47+
SRCS=(
48+
$ROOT/core/src/yamlstar/parser/prelude.clj
49+
$ROOT/core/src/yamlstar/parser/parser.clj
50+
$ROOT/core/src/yamlstar/parser/receiver.clj
51+
$ROOT/core/src/yamlstar/parser/grammar.clj
52+
$ROOT/core/src/yamlstar/parser.clj
53+
$ROOT/core/src/yamlstar/composer.clj
54+
$ROOT/core/src/yamlstar/resolver.clj
55+
$ROOT/core/src/yamlstar/constructor.clj
56+
$ROOT/core/src/yamlstar/core.clj
57+
$ROOT/bench/bench2.clj
58+
)
59+
60+
# Patch gloat's template go.mod to use the cloned glojure.
61+
# The template may not have a replace line, so append one if missing.
62+
for gomod in "$GLOAT_DIR/template/go.mod" "$GLOAT_DIR/ys/pkg/go.mod"; do
63+
if grep -q 'replace github.com/gloathub/glojure =>' "$gomod"; then
64+
sed -i "s|replace github.com/gloathub/glojure => .*|replace github.com/gloathub/glojure => $GLOJURE_BUILD|" "$gomod"
65+
else
66+
echo "replace github.com/gloathub/glojure => $GLOJURE_BUILD" >> "$gomod"
67+
fi
68+
done
69+
70+
echo
71+
echo "=== 1. Build glj from $COMMIT ==="
72+
time (cd "$GLOJURE_BUILD" && $GO build -o "$GLJ" ./cmd/glj)
73+
# Copy to gloat's glj path if different, so gloat uses our build
74+
if [[ -n "$GLOAT_GLJ" && "$GLOAT_GLJ" != "$GLJ" ]]; then
75+
echo "Copying glj to gloat path: $GLOAT_GLJ"
76+
cp "$GLJ" "$GLOAT_GLJ"
77+
touch "$GLOAT_GLJ"
78+
fi
79+
touch "$GLJ"
80+
81+
echo
82+
echo "=== 2. Compile to Go project ==="
83+
time (</dev/null env -u GOROOT "$GLOAT" "${SRCS[@]}" -o "$BUILD_DIR/" --force)
84+
85+
# Patch the generated go.mod to use the cloned glojure.
86+
if grep -q 'replace github.com/gloathub/glojure =>' "$BUILD_DIR/go.mod"; then
87+
sed -i "s|replace github.com/gloathub/glojure => .*|replace github.com/gloathub/glojure => $GLOJURE_BUILD|" \
88+
"$BUILD_DIR/go.mod"
89+
else
90+
echo "replace github.com/gloathub/glojure => $GLOJURE_BUILD" >> "$BUILD_DIR/go.mod"
91+
fi
92+
93+
echo
94+
echo "=== 3. Build binary ==="
95+
# Patch main.go to call bench2/-main instead of whatever gloat picked
96+
sed -i 's|Var("[^"]*", "-main")|Var("bench2", "-main")|' "$BUILD_DIR/main.go"
97+
sed -i 's|FindOrCreateNamespace(lang.NewSymbol("[^"]*"))|FindOrCreateNamespace(lang.NewSymbol("bench2"))|' "$BUILD_DIR/main.go"
98+
time (cd "$BUILD_DIR" && $GO mod tidy && $GO build -o "$BINARY" .)
99+
100+
echo
101+
echo "=== 4. Benchmark ==="
102+
MAX_LOAD=${MAX_LOAD:-2.0}
103+
while true; do
104+
load=$(awk '{print $1}' /proc/loadavg)
105+
if awk "BEGIN{exit !($load < $MAX_LOAD)}"; then
106+
break
107+
fi
108+
printf "Load average %.2f >= %.2f, waiting...\r" "$load" "$MAX_LOAD"
109+
sleep 5
110+
done
111+
echo "Load average $(awk '{print $1}' /proc/loadavg) — starting benchmark"
112+
"$BINARY"
113+
echo "Load average $(awk '{print $1}' /proc/loadavg) — benchmark complete"

‎bench/profile.clj‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
(ns profile
2+
(:require [yamlstar.core :as yaml]))
3+
4+
(def inputs
5+
[["scalar" "hello"]
6+
["mapping" "foo: 42"]
7+
["nested" "root:\n child1:\n key: value\n child2:\n - item1\n - item2\n - item3"]
8+
["types" "string: hello\ninteger: 42\nfloat: 3.14\nbool: true\nnull_val: null"]])
9+
10+
(defn now-ns []
11+
(.UnixNano (time.Now)))
12+
13+
(defn -main []
14+
;; Warmup — initialize all namespaces before profiling starts.
15+
;; The Go-side profiler wrapper starts AFTER this returns.
16+
(yaml/load "warmup: true")
17+
18+
(let [reps-str (os.Getenv "REPS")
19+
reps (if (not= reps-str "")
20+
(parse-long reps-str)
21+
5)]
22+
(println (fmt.Sprintf "Running %d iterations across 4 inputs (%d total parses)..."
23+
reps (* reps (count inputs))))
24+
25+
(doseq [[label input] inputs]
26+
(let [t0 (now-ns)]
27+
(dotimes [_ reps]
28+
(yaml/load input))
29+
(let [elapsed-ms (/ (double (- (now-ns) t0)) 1000000.0)
30+
per-ms (/ elapsed-ms reps)]
31+
(println (fmt.Sprintf " %-12s %7.1f ms total %7.1f ms/call"
32+
label elapsed-ms per-ms)))))
33+
34+
(println "Done.")))

0 commit comments

Comments
 (0)