|
| 1 | +#!/bin/bash |
| 2 | +# Usage: run_battery_binary.sh <model> <repeats> <conc> — runs the BUILT BINARY with --yolo |
| 3 | +MODEL="${1:-azure/gpt-5.5}"; REPEATS="${2:-1}"; CONC="${3:-1}" |
| 4 | +ROOT="$(git rev-parse --show-toplevel)" |
| 5 | +BIN="$ROOT/packages/opencode/dist/@altimateai/altimate-code-darwin-arm64/bin/altimate" |
| 6 | +TASKS="$ROOT/.github/meta/night-run/e2e/tasks.jsonl" |
| 7 | +OUT="/tmp/e2e_binary"; mkdir -p "$OUT"; RES="$OUT/res.d"; rm -rf "$RES"; mkdir -p "$RES"; : > "$OUT/results.tsv" |
| 8 | +run_one() { |
| 9 | + local id="$1" prompt="$2" check="$3" n="$4"; local wd; wd="$(mktemp -d /tmp/e2eb.XXXXXX)" |
| 10 | + ( cd "$wd" && git init -q 2>/dev/null; timeout 200 "$BIN" run "$prompt" --model "$MODEL" --yolo </dev/null >/dev/null 2>&1 |
| 11 | + if eval "$check" >/dev/null 2>&1; then v=PASS; else v=FAIL; fi |
| 12 | + printf '%s\t%s\t%s\n' "$id" "$n" "$v" > "$RES/${id}_${n}.tsv" ); rm -rf "$wd" |
| 13 | +} |
| 14 | +JOBS="$OUT/jobs.txt"; : > "$JOBS" |
| 15 | +while IFS= read -r line; do [ -z "$line" ] && continue |
| 16 | + id=$(printf '%s' "$line"|python3 -c "import json,sys;print(json.load(sys.stdin)['id'])") |
| 17 | + prompt=$(printf '%s' "$line"|python3 -c "import json,sys;print(json.load(sys.stdin)['prompt'])") |
| 18 | + check=$(printf '%s' "$line"|python3 -c "import json,sys;print(json.load(sys.stdin)['check'])") |
| 19 | + for r in $(seq 1 "$REPEATS"); do printf '%s\t%s\t%s\t%s\n' "$id" "$prompt" "$check" "$r" >> "$JOBS"; done |
| 20 | +done < "$TASKS" |
| 21 | +TOTAL=$(wc -l <"$JOBS"|tr -d ' '); echo "running $TOTAL binary e2e jobs (model=$MODEL --yolo conc=$CONC) $(date +%T)" |
| 22 | +while IFS=$'\t' read -r id prompt check n <&9; do |
| 23 | + run_one "$id" "$prompt" "$check" "$n" & |
| 24 | + while [ "$(jobs -rp|wc -l|tr -d ' ')" -ge "$CONC" ]; do sleep 0.3; done |
| 25 | +done 9< "$JOBS"; wait |
| 26 | +cat "$RES"/*.tsv > "$OUT/results.tsv" 2>/dev/null |
| 27 | +echo "=== RESULTS === PASS: $(grep -c $'\tPASS' "$OUT/results.tsv") FAIL: $(grep -c $'\tFAIL' "$OUT/results.tsv") / $TOTAL $(date +%T)" |
| 28 | +awk -F'\t' '{t[$1]++; if($3=="PASS")p[$1]++} END{for(k in t)printf " %-18s %d/%d\n",k,p[k],t[k]}' "$OUT/results.tsv"|sort |
0 commit comments